From 7fd321df9778b184d0acae59a08fd322c3f91649 Mon Sep 17 00:00:00 2001 From: Craig Williams Date: Fri, 9 Oct 2026 18:27:51 +0100 Subject: [PATCH] Added unit tests, documentation and function descriptions across the board. Increased cyclop complexity threshold to 15 which is a bit more sane than 10 and excluded _test.go files from linters. --- .golangci.json | 18 +- README.md | 153 ++++ internal/cli/fetch/once_test.go | 61 ++ internal/config/README.md | 133 ++++ internal/config/conf_logging.go | 3 + internal/config/conf_sections_test.go | 466 ++++++++++++ internal/config/section.go | 8 + internal/fetcher/fetcher_test.go | 309 ++++++++ internal/fetcher/worker_findATender.go | 5 + internal/fetcher/worker_ftsMock.go | 5 + internal/fetcher/worker_sleeper.go | 3 + internal/findATender/api_test.go | 796 ++++++++++++++++++++ internal/findATender/findATender.go | 13 + internal/findATender/findATender_test.go | 124 +++ internal/findATender/typeConvert.go | 18 + internal/findATender/typeConvert_test.go | 330 ++++++++ internal/logging/accesslog.go | 3 + internal/logging/accesslog_test.go | 138 ++++ internal/logging/logging.go | 8 + internal/logging/logging_test.go | 157 ++++ internal/mock/README.md | 115 +++ internal/mock/findATender/catalog.go | 5 + internal/mock/findATender/catalog_test.go | 134 ++++ internal/mock/findATender/generator.go | 10 + internal/mock/findATender/generator_test.go | 215 ++++++ internal/mock/findATender/handler.go | 2 + internal/mock/findATender/handler_test.go | 209 +++++ internal/mock/findATender/ratelimit_test.go | 134 ++++ internal/mock/findATender/store.go | 11 + internal/mock/findATender/store_test.go | 269 +++++++ internal/util/worker_test.go | 252 +++++++ 31 files changed, 4106 insertions(+), 1 deletion(-) create mode 100644 internal/cli/fetch/once_test.go create mode 100644 internal/config/README.md create mode 100644 internal/config/conf_sections_test.go create mode 100644 internal/fetcher/fetcher_test.go create mode 100644 internal/findATender/api_test.go create mode 100644 internal/findATender/findATender_test.go create mode 100644 internal/findATender/typeConvert_test.go create mode 100644 internal/logging/accesslog_test.go create mode 100644 internal/logging/logging_test.go create mode 100644 internal/mock/README.md create mode 100644 internal/mock/findATender/catalog_test.go create mode 100644 internal/mock/findATender/generator_test.go create mode 100644 internal/mock/findATender/handler_test.go create mode 100644 internal/mock/findATender/ratelimit_test.go create mode 100644 internal/mock/findATender/store_test.go create mode 100644 internal/util/worker_test.go diff --git a/.golangci.json b/.golangci.json index 6cb71a8..0472b6d 100644 --- a/.golangci.json +++ b/.golangci.json @@ -18,7 +18,23 @@ "nolintlint", "unused", "whitespace" - ] + ], + "settings": { + "cyclop": { + "max-complexity": 15 + } + }, + "exclusions": { + "rules": [ + { + "path": "_test\\.go", + "linters": [ + "errcheck", + "cyclop" + ] + } + ] + } }, "formatters": { "enable": [ diff --git a/README.md b/README.md index eb44e49..c36089d 100644 --- a/README.md +++ b/README.md @@ -3,6 +3,159 @@ JobSearch is a service for scraping for contract opportunities. It is primarily a re-write of the existing nodeJS JobSearch codebase. +It currently fetches releases from the UK Government +[Find a Tender](https://www.find-tender.service.gov.uk/) service (FTS), which +publishes them in the [OCDS](https://standard.open-contracting.org/) format. +A mock of the FTS API is included for development. + + +## Building +```bash +scripts/build.sh # build/job-search- +scripts/build.sh --release # also package release/job-search--.tar.gz +``` + +The build script runs `go generate ./...` and stamps the version from the +latest git tag. A build from untagged commits is marked as a development build +and isn't packaged unless `--force` is also given. + +During development, `go run ./cmd/jobsearch` works as well. + + +## Configuration +Configuration is read from, in increasing priority: +1. Built-in defaults +2. `config.yaml` in `/etc/job-search` or the working directory, or the file + given with `--config` +3. `JOBSEARCH_*` environment variables, e.g. `JOBSEARCH_LOGGING_LEVEL=debug` + +A `.env` file in the working directory is loaded into the environment first, +and must exist. + +Example `config.yaml`: +```yaml +logging: + level: info + file: /var/log/job-search.log +sources: + find_a_tender: + enabled: true + interval: 10m +``` + +See [internal/config](internal/config/README.md) for every key and how the +configuration code works. + + +## Usage +Global flags: `--config`/`-c` (config file), `--debug`/`-d` and +`--quiet`/`-q` (errors only). + +| Command | Purpose | +|---|---| +| `job-search fetch service` | Poll every enabled source on its interval until stopped (Ctrl+C or SIGTERM) | +| `job-search fetch once` | Fetch from Find a Tender once and print the releases as JSON | +| `job-search config init [path]` | Create a config file | +| `job-search config show` | Print the configuration (`--json` or `--yaml` for those formats) | +| `job-search config get ` | Print one value, e.g. `logging.level` | +| `job-search config set ` | Validate and write one value to the config file | +| `job-search config set --unset ` | Remove a value from the config file | +| `job-search version` | Print version information | + +`fetch service` logs to `logging.file` if it's set, or to the terminal with +`--foreground`. Both fetch commands take `--since "YYYY-MM-DD[ hh:mm:ss]"`. +`fetch once` can be limited with `--planning`, `--opportunities` or +`--awards`, and with `--ignore-rate-limit` it fails on a rate limit instead +of waiting. + + +## Mock Find a Tender server +To develop without using the real API, run the mock and enable the `fts_mock` +source: +```bash +go run ./cmd/mock-fts --min-gap 5s --max-gap 10s --rate-limit 1000 +``` +```yaml +sources: + fts_mock: + enabled: true + interval: 1m + api_root: http://localhost:8080/api/1.0 +``` + +See [internal/mock](internal/mock/README.md) for its options and behaviour. + + +## Project layout + +| Path | Contents | +|---|---| +| `cmd/jobsearch` | The `job-search` command | +| `cmd/mock-fts` | The mock FTS server | +| `internal/cli` | Cobra commands and shared CLI setup | +| `internal/config` | Configuration loading, validation and editing | +| `internal/fetcher` | Runs a worker per enabled source | +| `internal/findATender` | FTS API client and conversion from OCDS to JobSearch types | +| `internal/types` | JobSearch's own opportunity types | +| `internal/util` | The periodic `Worker` used by the fetcher | +| `internal/logging` | slog setup and the HTTP access log | +| `internal/mock` | Mock external services | +| `internal/version` | Version information set at build time | + + +## Testing +```bash +go test -race ./internal/... +go test -coverprofile=cover.out ./internal/... && go tool cover -func=cover.out +``` + +Tests don't need network access; HTTP calls go to local `httptest` servers. + + +## Addition of a new opportunities source +Using a source called `example` as an illustration: + +1. **Configuration section:** create `internal/config/conf_source_example.go`, + copying an existing source such as `conf_source_ftsMock.go`: + - Add a `JSCKSourceExample = "example"` constant. + - Define `JobSearchConfigSourceExample`, embedding `JobSearchConfigSource` + (for `enabled` and `interval`), plus any fields of its own with `cfg` and + optional `default` tags, e.g. ``APIRoot string `cfg:"api_root"` ``. + - Copy the standard `Map`, `Flatten`, `TypeOf`, `ValueOf` and `SetValue` + methods, renaming the receiver type and using the new constant in + `Flatten`. +2. **Register the section:** add + ``Example JobSearchConfigSourceExample `cfg:"example"` `` to + `JobSearchConfigSources` in `internal/config/conf_sources.go`. The `cfg` tag + must match the constant. The source's keys (`sources.example.*`) are then + loaded, validated and available to the `config` commands with no further + code. +3. **Client package:** create a package that fetches from the source and + returns `[]*types.Opportunity` (see `internal/types`). If the source + publishes OCDS, `internal/findATender` may already be suitable, as it is + for `fts_mock`: create it with the source's own API root. +4. **Worker:** create `internal/fetcher/worker_example.go`, following + `worker_ftsMock.go`: + - `runExample(ctx, conf, since)` builds a `util.Worker` with the source's + name, `conf.Sources.Example.Interval` and a task, then calls `Run`. + - The task fetches everything since its `since` argument. Wrap blocking + calls in `util.AwaitTask` so shutdown isn't held up, and return + `ErrNoNewReleases` when nothing is found, so the checkpoint still moves + forward. Any other error makes the next run retry from the same point. +5. **Start it:** in `Run` in `internal/fetcher/fetcher.go`, add + ```go + if conf.Sources.Example.Enabled { + start("Example", runExample) + } + ``` +6. **Tests:** add tests for the new config section to + `internal/config/conf_sections_test.go` (add it to + `TestConcreteSourceSections` and update the key count in + `TestSourcesSection`), and for the client and worker, using `httptest` + servers rather than the real service. +7. **Docs:** add the source and its keys to the key table in + `internal/config/README.md`. + ## Installation of dev tools A number of tools have been used in the development of the project. Installation diff --git a/internal/cli/fetch/once_test.go b/internal/cli/fetch/once_test.go new file mode 100644 index 0000000..9399e09 --- /dev/null +++ b/internal/cli/fetch/once_test.go @@ -0,0 +1,61 @@ +package fetch + +import ( + "io" + "slices" + "strings" + "testing" + + "git.avsdev.uk/Go/job-search/internal/cli" + "git.avsdev.uk/Go/job-search/internal/types" +) + +// opp makes an opportunity whose releases, oldest first, have these tags. +func opp(uuid string, tags ...string) *types.Opportunity { + o := &types.Opportunity{UUID: uuid} + for _, tag := range tags { + o.Releases = append(o.Releases, types.OpportunityRelease{ + Tags: []types.OpportunityReleaseTag{{Tag: tag}}, + }) + } + return o +} + +func TestFilter(t *testing.T) { + opps := []*types.Opportunity{ + opp("planned", "planning"), + opp("open", "planning", "tender"), + opp("extended", "tender", "tenderUpdate"), + opp("awarded", "planning", "tender", "award"), + opp("cancelled", "tender", "tenderCancellation"), + opp("no releases"), + } + tests := map[string][]string{ + "planning": {"planned"}, + "opportunity": {"open", "extended"}, + "award": {"awarded"}, + } + for stage, want := range tests { + t.Run(stage, func(t *testing.T) { + var got []string + for _, o := range filter(opps, stage) { + got = append(got, o.UUID) + } + if !slices.Equal(got, want) { + t.Errorf("filter(%s) = %v; want %v", stage, got, want) + } + }) + } +} + +// Conflicting filters are rejected by cobra before anything is fetched. +func TestOnceFiltersExclusive(t *testing.T) { + cmd := commandOnce(&cli.Options{}) + cmd.SetArgs([]string{"--planning", "--awards"}) + cmd.SetOut(io.Discard) + cmd.SetErr(io.Discard) + err := cmd.Execute() + if err == nil || !strings.Contains(err.Error(), "none of the others can be") { + t.Fatalf("err = %v; want a mutually exclusive flags error", err) + } +} diff --git a/internal/config/README.md b/internal/config/README.md new file mode 100644 index 0000000..83cebf4 --- /dev/null +++ b/internal/config/README.md @@ -0,0 +1,133 @@ +# config + +Package config loads the JobSearch configuration into a `JobSearchConfig` +struct. Values come from struct-tag defaults, a YAML or JSON config file and +environment variables, merged through [Viper](https://github.com/spf13/viper). + + +## Loading order +`Init(cfgFile, &conf)` builds the configuration in this order: + +1. Loads `.env` from the working directory into the environment. A missing + `.env` file is an error. +2. Sets the defaults from the `default` struct tags as Viper defaults. +3. Reads the config file: `cfgFile` if given, otherwise the first + `config.yaml` found in `/etc/job-search` then the working directory. If no + file is found a warning is logged and the defaults are used. A file that + exists but can't be parsed is an error. +4. Reads environment variables prefixed with `JOBSEARCH_`. Dots in a key + become underscores, so `logging.level` is read from + `JOBSEARCH_LOGGING_LEVEL`. +5. Copies every known key from Viper into `conf`. A value that fails + validation (e.g. an unknown log level) is logged as a warning and the + previous value is kept. + +When more than one place sets a key, environment variables win over the +config file, which wins over the defaults. A `debug` or `error` log level +already set on `conf` before `Init` (from the `--debug` and `--quiet` flags) +is kept. + + +## Keys + +| Key | Type | Default | Notes | +|---|---|---|---| +| `logging.level` | string | `warn` | One of `debug`, `info`, `warn`, `error` | +| `logging.file` | string | | Log file for `fetch service` (unless `--foreground`) | +| `sources..enabled` | bool | `false` | For every source | +| `sources..interval` | duration | `10m` | For every source; e.g. `30s`, `1h` | +| `sources.find_a_tender.api_root` | string | URL of the Find A Tender service API. Currently `https://www.find-tender.service.gov.uk/api/1.0` | | +| `sources.fts_mock.api_root` | string | | URL of the mock FTS server (see `internal/mock`) | +| `sources.sleeper.block_shutdown` | bool | `false` | Test source: if true, its sleep ignores shutdown | + +The current sources are `find_a_tender`, `fts_mock` and `sleeper`. See +"Addition of a new opportunities source" in the top level README for how to add +a new source. + +Example `config.yaml`: +```yaml +logging: + level: info + file: "" +sources: + find_a_tender: + enabled: true + interval: 10m + fts_mock: + enabled: false + interval: 1m + api_root: http://localhost:8080/api/1.0 +``` + + +## Sections and struct tags +Each section of the configuration is a plain struct. The reflection code in +`section.go` reads its struct tags, so a section needs no hand-written code +per key: + +- `cfg:"key"` is the field's configuration key. Fields without it are not part + of the configuration. +- `default:"value"` is the field's default, parsed as if it were a string + passed to `SetValue` (e.g. `"10m"`, `"true"`). +- A tagged field whose type is a struct is a nested section, addressed as + `section.key`. An embedded struct has its fields promoted, as Go does; + every source embeds `JobSearchConfigSource` for `enabled` and `interval`. +- `time.Duration` fields report `reflect.String` from `TypeOf`, accept either a + `time.Duration` or a duration string in `SetValue`, and are written as + strings (`"10m0s"`) by `Map`, `JSON` and `YAML`. +- A section can reject values by implementing `validate(key, value) error` on + its pointer type. `JobSearchConfigLogging` does this for `level`. + +Each section type has the same small set of methods, each a one-line call into +`section.go`: `Map`, `Flatten`, `TypeOf`, `ValueOf` and `SetValue`, plus +`Keys` on some. + + +## Adding a key or section +To add a key to an existing section, add a tagged field to its struct. Nothing +else is needed for it to be loaded, printed, and read or written by the +`config` CLI commands. + +To add a new section: +1. Create a `conf_*.go` file with the section struct, a `JSCK*` constant for + its key, and the standard methods (copy an existing section). +2. Add the struct as a tagged field of its parent: `JobSearchConfig` in + `config.go`, or `JobSearchConfigSources` in `conf_sources.go` for a source, + etc. The field's `cfg` tag must match the `JSCK*` constant. + + +## API +- `Init(cfgFile, &conf)`: load the configuration, as described above. +- `conf.TypeOf(key)`, `conf.ValueOf(key)`, `conf.SetValue(key, value)`: read + or set one dotted key on the loaded struct. `SetValue` converts and + validates the value. +- `conf.Map()` (nested) and `conf.Flatten()` (dotted keys): the whole + configuration as a map. +- `conf.String()`, `conf.JSON()`, `conf.YAML()` and the matching `Print*` + methods: the configuration as text. +- `StoreValue(&conf, key, value)` / `PurgeValue(&conf, key)`: set or remove a + key in the config file Viper loaded. Used by `config set` and + `config set --unset`. `StoreValue` validates the value before writing it. +- `LoadDefaultConfig`, `LoadToViper`, `LoadFromViper`: move values between + the struct defaults, Viper and a `JobSearchConfig`. + + +## Files + +| File | Contents | +|---|---| +| `config.go` | `JobSearchConfig`, defaults, Viper loading, text output | +| `load.go` | `Init`, `.env` and config file loading | +| `modify.go` | `StoreValue` and `PurgeValue` | +| `section.go` | Struct tag reflection shared by every section | +| `conf_logging.go` | `logging` section and log level helpers | +| `conf_sources.go` | `sources` section and the shared source fields | +| `conf_source_*.go` | One file per source section | +| `print.go`, `debug.go` | Printing and debug logging of the configuration | + + +## Testing +Viper keeps its state in package globals, so tests that use it call +`resetViper(t)` to reset it before and after the test. Tests of `Init` +use `initDir`, which changes into a temp directory with its own `.env` and +`config.yaml`, so the real files in the repository are never read. diff --git a/internal/config/conf_logging.go b/internal/config/conf_logging.go index db9ccdc..23d254b 100644 --- a/internal/config/conf_logging.go +++ b/internal/config/conf_logging.go @@ -67,6 +67,9 @@ func ConfigLevelToLogLevel(level string) slog.Level { return 99 } } + +// LogLevel returns the configured level as an slog level. An unknown level +// gives 99, above every real level, so nothing is logged. func (c *JobSearchConfigLogging) LogLevel() slog.Level { return ConfigLevelToLogLevel(c.Level) } diff --git a/internal/config/conf_sections_test.go b/internal/config/conf_sections_test.go new file mode 100644 index 0000000..bf70c56 --- /dev/null +++ b/internal/config/conf_sections_test.go @@ -0,0 +1,466 @@ +package config + +import ( + "bytes" + "encoding/json" + "log/slog" + "reflect" + "strings" + "testing" + "time" + + "github.com/spf13/viper" + "go.yaml.in/yaml/v3" +) + +// resetViper isolates tests that touch the global Viper instance. +func resetViper(t *testing.T) { + t.Helper() + viper.Reset() + t.Cleanup(viper.Reset) +} + +// captureLogs sends the default slog logger to a buffer until the test ends. +func captureLogs(t *testing.T, level slog.Level) *bytes.Buffer { + t.Helper() + var buf bytes.Buffer + def := slog.Default() + slog.SetDefault(slog.New(slog.NewTextHandler(&buf, &slog.HandlerOptions{Level: level}))) + t.Cleanup(func() { slog.SetDefault(def) }) + return &buf +} + +func defaultConfig(t *testing.T) JobSearchConfig { + t.Helper() + var conf JobSearchConfig + if err := applyDefaults(&conf); err != nil { + t.Fatalf("defaults: %v", err) + } + return conf +} + +func TestDefaults(t *testing.T) { + conf := defaultConfig(t) + if conf.Logging.Level != "warn" { + t.Errorf("Logging.Level = %q; want warn", conf.Logging.Level) + } + if conf.Sources.FindATender.APIRoot != "https://www.find-tender.service.gov.uk/api/1.0" { + t.Errorf("FindATender.APIRoot = %q", conf.Sources.FindATender.APIRoot) + } + if conf.Sources.FTSMock.APIRoot != "" { + t.Errorf("FTSMock.APIRoot = %q; want empty", conf.Sources.FTSMock.APIRoot) + } + for name, src := range map[string]JobSearchConfigSource{ + "find_a_tender": conf.Sources.FindATender.JobSearchConfigSource, + "fts_mock": conf.Sources.FTSMock.JobSearchConfigSource, + "sleeper": conf.Sources.Sleeper.JobSearchConfigSource, + } { + if src.Enabled || src.Interval != 10*time.Minute { + t.Errorf("%s source = %+v; want disabled, 10m", name, src) + } + } +} + +func TestLoggingSection(t *testing.T) { + c := JobSearchConfigLogging{Level: "info", File: "/tmp/x.log"} + + if got, want := c.Keys(), []string{"level", "file"}; !reflect.DeepEqual(got, want) { + t.Errorf("Keys = %v; want %v", got, want) + } + if got, want := c.Map(), map[string]any{ + "level": "info", + "file": "/tmp/x.log", + }; !reflect.DeepEqual( + got, + want, + ) { + t.Errorf("Map = %v; want %v", got, want) + } + if got, want := c.Flatten(), map[string]any{ + "logging.level": "info", + "logging.file": "/tmp/x.log", + }; !reflect.DeepEqual( + got, + want, + ) { + t.Errorf("Flatten = %v; want %v", got, want) + } + if k, err := c.TypeOf("file"); err != nil || k != reflect.String { + t.Errorf("TypeOf(file) = %v, %v", k, err) + } + if v, err := c.ValueOf("level"); err != nil || v != "info" { + t.Errorf("ValueOf(level) = %v, %v", v, err) + } + if _, err := c.TypeOf("nope"); err == nil { + t.Error("TypeOf(nope) succeeded") + } + if err := c.SetValue("file", "/var/log/js.log"); err != nil || c.File != "/var/log/js.log" { + t.Errorf("SetValue(file) = %v, File = %q", err, c.File) + } +} + +func TestLoggingLevelValidation(t *testing.T) { + var c JobSearchConfigLogging + for _, lvl := range []string{"debug", "info", "warn", "error"} { + if err := c.SetValue("level", lvl); err != nil || c.Level != lvl { + t.Errorf("SetValue(level, %q) = %v, Level = %q", lvl, err, c.Level) + } + } + for _, lvl := range []string{"", "trace", "WARN", "fatal"} { + if err := c.SetValue( + "level", + lvl, + ); err == nil || + !strings.Contains(err.Error(), "invalid log level") { + t.Errorf("SetValue(level, %q) = %v; want invalid log level", lvl, err) + } + } + if c.Level != "error" { + t.Errorf("rejected level was stored: %q", c.Level) + } + // Validation also applies through the top-level config. + conf := defaultConfig(t) + if err := conf.SetValue("logging.level", "loud"); err == nil { + t.Error("JobSearchConfig.SetValue(logging.level, loud) succeeded") + } + // Other keys are not validated. + if err := c.validate("file", "anything"); err != nil { + t.Errorf("validate(file) = %v", err) + } +} + +func TestConfigLevelToLogLevel(t *testing.T) { + tests := map[string]slog.Level{ + "debug": slog.LevelDebug, + "info": slog.LevelInfo, + "warn": slog.LevelWarn, + "error": slog.LevelError, + "": 99, + "bogus": 99, + } + for in, want := range tests { + if got := ConfigLevelToLogLevel(in); got != want { + t.Errorf("ConfigLevelToLogLevel(%q) = %v; want %v", in, got, want) + } + c := JobSearchConfigLogging{Level: in} + if got := c.LogLevel(); got != want { + t.Errorf("LogLevel() with %q = %v; want %v", in, got, want) + } + } +} + +func TestSourceSection(t *testing.T) { + c := JobSearchConfigSource{Enabled: true, Interval: time.Minute} + + if got, want := c.Keys(), []string{"enabled", "interval"}; !reflect.DeepEqual(got, want) { + t.Errorf("Keys = %v; want %v", got, want) + } + want := map[string]any{"enabled": true, "interval": "1m0s"} + if got := c.Map(); !reflect.DeepEqual(got, want) { + t.Errorf("Map = %v; want %v", got, want) + } + wantFlat := map[string]any{"enabled": true, "interval": time.Minute} + if got := c.Flatten(); !reflect.DeepEqual(got, wantFlat) { + t.Errorf("Flatten = %v; want %v (no prefix)", got, wantFlat) + } + if k, err := c.TypeOf("interval"); err != nil || k != reflect.String { + t.Errorf("TypeOf(interval) = %v, %v; want String", k, err) + } + if k, err := c.TypeOf("enabled"); err != nil || k != reflect.Bool { + t.Errorf("TypeOf(enabled) = %v, %v; want Bool", k, err) + } + if v, err := c.ValueOf("interval"); err != nil || v != time.Minute { + t.Errorf("ValueOf(interval) = %v, %v", v, err) + } + if err := c.SetValue("interval", "5s"); err != nil || c.Interval != 5*time.Second { + t.Errorf("SetValue(interval, 5s) = %v, Interval = %v", err, c.Interval) + } + if err := c.SetValue("interval", "never"); err == nil { + t.Error("SetValue(interval, never) succeeded") + } +} + +// sourceSection is the method set shared by every concrete source section. +type sourceSection interface { + Map() map[string]any + Flatten() map[string]any + TypeOf(string) (reflect.Kind, error) + ValueOf(string) (any, error) + SetValue(string, any) error +} + +func TestConcreteSourceSections(t *testing.T) { + tests := []struct { + prefix string + section sourceSection + apiRoot bool + blockShutdown bool + }{ + {JSCKSourceFindATender, &JobSearchConfigSourceFindATender{}, true, false}, + {JSCKSourceFTSMock, &JobSearchConfigSourceFTSMock{}, true, false}, + {JSCKSourceSleeper, &JobSearchConfigSourceSleeper{}, false, true}, + } + for _, tt := range tests { + t.Run(tt.prefix, func(t *testing.T) { + s := tt.section + // Embedded source fields are promoted. + if err := s.SetValue("enabled", "true"); err != nil { + t.Fatalf("SetValue(enabled): %v", err) + } + if err := s.SetValue("interval", 2*time.Minute); err != nil { + t.Fatalf("SetValue(interval): %v", err) + } + if v, err := s.ValueOf("enabled"); err != nil || v != true { + t.Errorf("ValueOf(enabled) = %v, %v", v, err) + } + if k, err := s.TypeOf("interval"); err != nil || k != reflect.String { + t.Errorf("TypeOf(interval) = %v, %v", k, err) + } + + wantMap := map[string]any{"enabled": true, "interval": "2m0s"} + wantFlat := map[string]any{ + tt.prefix + ".enabled": true, + tt.prefix + ".interval": 2 * time.Minute, + } + if tt.apiRoot { + if err := s.SetValue("api_root", "http://localhost:8080"); err != nil { + t.Fatalf("SetValue(api_root): %v", err) + } + if v, err := s.ValueOf("api_root"); err != nil || v != "http://localhost:8080" { + t.Errorf("ValueOf(api_root) = %v, %v", v, err) + } + wantMap["api_root"] = "http://localhost:8080" + wantFlat[tt.prefix+".api_root"] = "http://localhost:8080" + } else if _, err := s.TypeOf("api_root"); err == nil { + t.Error("TypeOf(api_root) succeeded on a source without one") + } + if tt.blockShutdown { + if err := s.SetValue("block_shutdown", "true"); err != nil { + t.Fatalf("SetValue(block_shutdown): %v", err) + } + if k, err := s.TypeOf("block_shutdown"); err != nil || k != reflect.Bool { + t.Errorf("TypeOf(block_shutdown) = %v, %v", k, err) + } + wantMap["block_shutdown"] = true + wantFlat[tt.prefix+".block_shutdown"] = true + } else if _, err := s.TypeOf("block_shutdown"); err == nil { + t.Error("TypeOf(block_shutdown) succeeded on a source without one") + } + + if got := s.Map(); !reflect.DeepEqual(got, wantMap) { + t.Errorf("Map = %v; want %v", got, wantMap) + } + if got := s.Flatten(); !reflect.DeepEqual(got, wantFlat) { + t.Errorf("Flatten = %v; want %v", got, wantFlat) + } + if _, err := s.ValueOf("nope"); err == nil { + t.Error("ValueOf(nope) succeeded") + } + }) + } +} + +func TestSourcesSection(t *testing.T) { + var c JobSearchConfigSources + if err := c.SetValue("fts_mock.api_root", "http://mock"); err != nil { + t.Fatalf("SetValue: %v", err) + } + if c.FTSMock.APIRoot != "http://mock" { + t.Errorf("FTSMock.APIRoot = %q", c.FTSMock.APIRoot) + } + if v, err := c.ValueOf("fts_mock.api_root"); err != nil || v != "http://mock" { + t.Errorf("ValueOf = %v, %v", v, err) + } + if k, err := c.TypeOf("sleeper.enabled"); err != nil || k != reflect.Bool { + t.Errorf("TypeOf(sleeper.enabled) = %v, %v", k, err) + } + if _, err := c.TypeOf("sleeper"); err == nil { + t.Error("TypeOf(sleeper) succeeded on a section key") + } + + m := c.Map() + for _, k := range []string{"find_a_tender", "fts_mock", "sleeper"} { + if _, ok := m[k].(map[string]any); !ok { + t.Errorf("Map()[%q] = %#v; want nested map", k, m[k]) + } + } + flat := c.Flatten() + if flat["sources.fts_mock.api_root"] != "http://mock" { + t.Errorf("Flatten missing sources.fts_mock.api_root: %v", flat) + } + if len(flat) != 9 { + t.Errorf("Flatten has %d keys; want 9: %v", len(flat), flat) + } +} + +func TestJobSearchConfigMap(t *testing.T) { + conf := defaultConfig(t) + m := conf.Map() + logging, ok := m["logging"].(map[string]any) + if !ok || logging["level"] != "warn" { + t.Errorf("Map()[logging] = %#v", m["logging"]) + } + sources, ok := m["sources"].(map[string]any) + if !ok { + t.Fatalf("Map()[sources] = %#v", m["sources"]) + } + fat, ok := sources["find_a_tender"].(map[string]any) + if !ok || fat["interval"] != "10m0s" { + t.Errorf("Map()[sources][find_a_tender] = %#v", sources["find_a_tender"]) + } +} + +func TestString(t *testing.T) { + conf := defaultConfig(t) + s := conf.String() + for _, want := range []string{ + "logging.level = warn \n", + "sources.sleeper.enabled = false \n", + "sources.find_a_tender.interval = 10m0s \n", + } { + if !strings.Contains(s, want) { + t.Errorf("String() missing %q:\n%s", want, s) + } + } + if n := strings.Count(s, "\n"); n != len(conf.Flatten()) { + t.Errorf("String() has %d lines; want %d", n, len(conf.Flatten())) + } +} + +func TestJSON(t *testing.T) { + conf := defaultConfig(t) + var m map[string]map[string]any + if err := json.Unmarshal([]byte(conf.JSON()), &m); err != nil { + t.Fatalf("JSON() is not valid JSON: %v\n%s", err, conf.JSON()) + } + if m["logging"]["level"] != "warn" { + t.Errorf("JSON logging.level = %#v", m["logging"]["level"]) + } + if _, ok := m["sources"]["fts_mock"]; !ok { + t.Errorf("JSON missing sources.fts_mock: %v", m["sources"]) + } +} + +func TestYAML(t *testing.T) { + conf := defaultConfig(t) + var m map[string]map[string]any + if err := yaml.Unmarshal([]byte(conf.YAML()), &m); err != nil { + t.Fatalf("YAML() is not valid YAML: %v\n%s", err, conf.YAML()) + } + if m["logging"]["level"] != "warn" { + t.Errorf("YAML logging.level = %#v", m["logging"]["level"]) + } + if _, ok := m["sources"]["sleeper"]; !ok { + t.Errorf("YAML missing sources.sleeper: %v", m["sources"]) + } +} + +// Output of JSON()/YAML() can be read back as a config file. +func TestSerialisedRoundTrip(t *testing.T) { + orig := defaultConfig(t) + orig.Logging.Level = "info" + orig.Sources.FTSMock.Enabled = true + orig.Sources.FTSMock.APIRoot = "http://mock" + for _, format := range []string{"json", "yaml"} { + t.Run(format, func(t *testing.T) { + resetViper(t) + out := orig.JSON() + if format == "yaml" { + out = orig.YAML() + } + viper.SetConfigType(format) + if err := viper.ReadConfig(strings.NewReader(out)); err != nil { + t.Fatalf("ReadConfig: %v", err) + } + var got JobSearchConfig + LoadFromViper(&got) + if !reflect.DeepEqual(got, orig) { + t.Errorf("round trip =\n%+v\nwant\n%+v", got, orig) + } + }) + } +} + +func TestLoadToViper(t *testing.T) { + resetViper(t) + conf := defaultConfig(t) + LoadToViper(conf) + for key, want := range conf.Flatten() { + if got := viper.Get(key); !reflect.DeepEqual(got, want) { + t.Errorf("viper.Get(%q) = %#v; want %#v", key, got, want) + } + } +} + +func TestLoadDefaultConfig(t *testing.T) { + resetViper(t) + LoadDefaultConfig() + if got := viper.GetString("logging.level"); got != "warn" { + t.Errorf("logging.level = %q; want warn", got) + } + if got := viper.GetDuration("sources.sleeper.interval"); got != 10*time.Minute { + t.Errorf("sources.sleeper.interval = %v; want 10m", got) + } +} + +func TestLoadFromViper(t *testing.T) { + resetViper(t) + LoadDefaultConfig() + viper.Set("logging.level", "debug") + viper.Set("logging.file", "/tmp/js.log") + viper.Set("sources.sleeper.enabled", "true") // string, read as bool + viper.Set("sources.fts_mock.interval", "30s") + viper.Set("sources.fts_mock.api_root", "http://mock") + viper.Set("unknown.key", "ignored") + + var conf JobSearchConfig + LoadFromViper(&conf) + + want := defaultConfig(t) + want.Logging.Level = "debug" + want.Logging.File = "/tmp/js.log" + want.Sources.Sleeper.Enabled = true + want.Sources.FTSMock.Interval = 30 * time.Second + want.Sources.FTSMock.APIRoot = "http://mock" + if !reflect.DeepEqual(conf, want) { + t.Errorf("LoadFromViper =\n%+v\nwant\n%+v", conf, want) + } +} + +func TestViperRoundTrip(t *testing.T) { + resetViper(t) + orig := defaultConfig(t) + orig.Logging.Level = "error" + orig.Sources.FindATender.Enabled = true + orig.Sources.FindATender.Interval = 45 * time.Second + LoadToViper(orig) + + var got JobSearchConfig + LoadFromViper(&got) + if !reflect.DeepEqual(got, orig) { + t.Errorf("round trip =\n%+v\nwant\n%+v", got, orig) + } +} + +// Invalid values from Viper are skipped with a warning, leaving the existing +// value. +func TestLoadFromViperInvalidValues(t *testing.T) { + resetViper(t) + logs := captureLogs(t, slog.LevelWarn) + viper.Set("logging.level", "shouty") + viper.Set("sources.sleeper.interval", "eventually") + + conf := defaultConfig(t) + LoadFromViper(&conf) + if conf.Logging.Level != "warn" { + t.Errorf("Logging.Level = %q; want warn kept", conf.Logging.Level) + } + if conf.Sources.Sleeper.Interval != 10*time.Minute { + t.Errorf("Sleeper.Interval = %v; want 10m kept", conf.Sources.Sleeper.Interval) + } + for _, key := range []string{"logging.level", "sources.sleeper.interval"} { + if !strings.Contains(logs.String(), "key="+key) { + t.Errorf("no warning logged for %s: %s", key, logs) + } + } +} diff --git a/internal/config/section.go b/internal/config/section.go index d3122d6..55d7333 100644 --- a/internal/config/section.go +++ b/internal/config/section.go @@ -205,6 +205,9 @@ func setValue(ptr any, key string, value any) error { return setField(section, f, value) } +// setField converts value to the field's type and, if the section has a +// validator, checks it, before setting the field. On error the field is left +// unchanged. func setField(section reflect.Value, f field, value any) error { newValue, err := convert(value, f.typ) if err != nil { @@ -234,6 +237,8 @@ func convert(value any, t reflect.Type) (reflect.Value, error) { return reflect.Value{}, fmt.Errorf("unsupported type %T", value) } +// parseString parses s as a value of type t. It supports strings, bools, +// signed and unsigned integers, floats and time.Duration. func parseString(s string, t reflect.Type) (reflect.Value, error) { v := reflect.New(t).Elem() if t == durationType { @@ -280,6 +285,9 @@ func applyDefaults(ptr any) error { return applyDefaultsTo("", structValue(ptr)) } +// applyDefaultsTo sets every field in section that has a default tag, +// recursing into nested sections. prefix is the section's dotted key, used in +// error messages. func applyDefaultsTo(prefix string, section reflect.Value) error { for _, f := range fieldsOf(section.Type()) { key := joinKey(prefix, f.key) diff --git a/internal/fetcher/fetcher_test.go b/internal/fetcher/fetcher_test.go new file mode 100644 index 0000000..a49f017 --- /dev/null +++ b/internal/fetcher/fetcher_test.go @@ -0,0 +1,309 @@ +package fetcher + +import ( + "bytes" + "context" + "errors" + "io" + "log/slog" + "net/http" + "net/http/httptest" + "os" + "strings" + "sync/atomic" + "testing" + "time" + + "git.avsdev.uk/Go/job-search/internal/config" + "git.avsdev.uk/Go/job-search/internal/util" +) + +func TestMain(m *testing.M) { + // Keep worker and fetch log lines out of the test output. + slog.SetDefault(slog.New(slog.NewTextHandler(io.Discard, nil))) + os.Exit(m.Run()) +} + +const onePackage = `{ + "uri": "x", + "releases": [{ + "ocid": "ocds-1", + "id": "r1", + "date": "2026-10-01T12:00:00Z", + "tag": ["tender"], + "tender": {"id": "t1", "title": "A tender"} + }] +}` + +const emptyPackage = `{"uri": "x", "releases": []}` + +// ftsServer serves body (with status) for every request and counts requests. +func ftsServer(t *testing.T, status int, body string) (*httptest.Server, *atomic.Int32) { + t.Helper() + var hits atomic.Int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + hits.Add(1) + if r.URL.Path != "/ocdsReleasePackages" { + t.Errorf("unexpected path %q", r.URL.Path) + } + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + io.WriteString(w, body) + })) + t.Cleanup(srv.Close) + return srv, &hits +} + +func confFor(apiRoot string) *config.JobSearchConfig { + conf := &config.JobSearchConfig{} + conf.Sources.FindATender.APIRoot = apiRoot + conf.Sources.FTSMock.APIRoot = apiRoot + return conf +} + +func TestFetchFunctions(t *testing.T) { + fetchers := map[string]func(context.Context, *config.JobSearchConfig, time.Time) error{ + "FindAOpportunity": fetchFindAOpportunity, + "FTSMock": fetchFTSMock, + } + tests := []struct { + name string + status int + body string + check func(error) bool + wantErr string + }{ + { + "releases found", + http.StatusOK, + onePackage, + func(err error) bool { return err == nil }, + "nil", + }, + {"no releases", http.StatusOK, emptyPackage, func(err error) bool { + return errors.Is(err, ErrNoNewReleases) && errors.Is(err, util.ErrWorkerNoWork) + }, "ErrNoNewReleases wrapping ErrWorkerNoWork"}, + {"bad body", http.StatusInternalServerError, "oops", func(err error) bool { + return err != nil && !errors.Is(err, util.ErrWorkerNoWork) + }, "a failure error"}, + } + for fname, fetch := range fetchers { + for _, tc := range tests { + t.Run(fname+"/"+tc.name, func(t *testing.T) { + srv, hits := ftsServer(t, tc.status, tc.body) + err := fetch(context.Background(), confFor(srv.URL), time.Now().Add(-time.Hour)) + if !tc.check(err) { + t.Fatalf("err = %v, want %s", err, tc.wantErr) + } + if hits.Load() == 0 { + t.Fatal("server was not called") + } + }) + } + } +} + +func TestFetchUsesConfiguredAPIRoot(t *testing.T) { + fat, fatHits := ftsServer(t, http.StatusOK, emptyPackage) + mock, mockHits := ftsServer(t, http.StatusOK, emptyPackage) + conf := &config.JobSearchConfig{} + conf.Sources.FindATender.APIRoot = fat.URL + conf.Sources.FTSMock.APIRoot = mock.URL + + fetchFindAOpportunity(context.Background(), conf, time.Time{}) + if fatHits.Load() != 1 || mockHits.Load() != 0 { + t.Fatalf("FindAOpportunity hits: fat=%d mock=%d", fatHits.Load(), mockHits.Load()) + } + fetchFTSMock(context.Background(), conf, time.Time{}) + if fatHits.Load() != 1 || mockHits.Load() != 1 { + t.Fatalf("FTSMock hits: fat=%d mock=%d", fatHits.Load(), mockHits.Load()) + } +} + +func TestFetchCancelled(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + // Unroutable root: the request should never be waited on. + conf := confFor("http://192.0.2.1:9") + start := time.Now() + if err := fetchFTSMock(ctx, conf, time.Time{}); !errors.Is(err, context.Canceled) { + t.Fatalf("err = %v, want context.Canceled", err) + } + if d := time.Since(start); d > time.Second { + t.Fatalf("cancelled fetch took %v", d) + } +} + +func TestSleepCancelled(t *testing.T) { + for _, block := range []bool{false, true} { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Millisecond) + defer cancel() + if err := sleep(ctx, block, time.Hour); !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("block=%v: err = %v, want context.DeadlineExceeded", block, err) + } + } +} + +func TestSleepCompletes(t *testing.T) { + for _, block := range []bool{false, true} { + if err := sleep(context.Background(), block, time.Millisecond); err != nil { + t.Fatalf("block=%v: err = %v", block, err) + } + } +} + +// A non-blocking sleeper stops as soon as shutdown is signalled; a blocking +// one holds shutdown open until the worker's shutdown timeout. +func TestRunSleeperShutdown(t *testing.T) { + for _, block := range []bool{false, true} { + conf := &config.JobSearchConfig{} + conf.Sources.Sleeper.Interval = time.Hour + conf.Sources.Sleeper.BlockShutdown = block + + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan struct{}) + go func() { + runSleeper(ctx, conf, time.Time{}) + close(done) + }() + time.Sleep(10 * time.Millisecond) // let the first sleep start + cancel() + + select { + case <-done: + if block { + t.Fatal("block=true: shutdown was not held open") + } + case <-time.After(time.Second): + if !block { + t.Fatal("block=false: shutdown was held open") + } + } + } +} + +func TestRunWorkersRejectZeroInterval(t *testing.T) { + conf := &config.JobSearchConfig{} + runs := map[string]func(context.Context, *config.JobSearchConfig, time.Time) error{ + "FindAOpportunity": runFindAOpportunity, + "FTSMock": runFTSMock, + "Sleeper": runSleeper, + } + for name, run := range runs { + if err := run(context.Background(), conf, time.Time{}); err == nil { + t.Errorf("%s: expected an error for a zero interval", name) + } + } +} + +// A source that stops with an error has the error logged. +func TestRunLogsSourceErrors(t *testing.T) { + var buf bytes.Buffer + def := slog.Default() + slog.SetDefault(slog.New(slog.NewTextHandler(&buf, nil))) + t.Cleanup(func() { slog.SetDefault(def) }) + + // A zero interval makes each worker fail straight away. + conf := &config.JobSearchConfig{} + conf.Sources.FindATender.Enabled = true + conf.Sources.FTSMock.Enabled = true + conf.Sources.Sleeper.Enabled = true + err := Run(context.Background(), conf, time.Time{}) + + for _, name := range []string{"FindATender", "FTSMock", "Sleeper"} { + if !strings.Contains(buf.String(), "source="+name) { + t.Errorf("no error logged for %s: %s", name, &buf) + } + if err == nil || !strings.Contains(err.Error(), name+": ") { + t.Errorf("returned error %v; want it to include %s", err, name) + } + } + if n := strings.Count(buf.String(), "interval must be greater than 0"); n != 3 { + t.Errorf("logged %d interval errors; want 3: %s", n, &buf) + } +} + +func TestRunNoSourcesEnabled(t *testing.T) { + done := make(chan struct{}) + go func() { + if err := Run(context.Background(), &config.JobSearchConfig{}, time.Time{}); err != nil { + t.Errorf("Run = %v; want nil", err) + } + close(done) + }() + select { + case <-done: + case <-time.After(time.Second): + t.Fatal("Run with no sources enabled did not return") + } +} + +// A worker that fails shuts the other workers down, without ctx being +// cancelled. +func TestRunShutsDownOnWorkerError(t *testing.T) { + mock, mockHits := ftsServer(t, http.StatusOK, emptyPackage) + conf := &config.JobSearchConfig{} + conf.Sources.FTSMock.Enabled = true + conf.Sources.FTSMock.Interval = time.Hour + conf.Sources.FTSMock.APIRoot = mock.URL + // The sleeper fails straight away: a zero interval is invalid. + conf.Sources.Sleeper.Enabled = true + + done := make(chan error, 1) + go func() { done <- Run(context.Background(), conf, time.Now().Add(-time.Hour)) }() + + select { + case err := <-done: + if err == nil || !strings.Contains(err.Error(), "Sleeper: ") { + t.Fatalf("Run = %v; want the Sleeper error", err) + } + if strings.Contains(err.Error(), "FTSMock") { + t.Errorf("Run = %v; FTSMock should have stopped cleanly", err) + } + case <-time.After(5 * time.Second): + t.Fatal("Run did not shut down after a worker failed") + } + // FTSMock either polled once before the shutdown or never started a run. + if n := mockHits.Load(); n > 1 { + t.Errorf("FTSMock polled %d times; want at most 1", n) + } +} + +// Run starts each enabled source, and returns once ctx is cancelled. +func TestRunEnabledSources(t *testing.T) { + fat, fatHits := ftsServer(t, http.StatusOK, onePackage) + mock, mockHits := ftsServer(t, http.StatusOK, emptyPackage) + conf := &config.JobSearchConfig{} + conf.Sources.FindATender.Enabled = true + conf.Sources.FindATender.Interval = time.Hour + conf.Sources.FindATender.APIRoot = fat.URL + conf.Sources.FTSMock.Enabled = true + conf.Sources.FTSMock.Interval = time.Hour + conf.Sources.FTSMock.APIRoot = mock.URL + conf.Sources.Sleeper.Enabled = true + conf.Sources.Sleeper.Interval = time.Hour + + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan struct{}) + go func() { + if err := Run(ctx, conf, time.Now().Add(-time.Hour)); err != nil { + t.Errorf("Run = %v; want nil after cancellation", err) + } + close(done) + }() + + deadline := time.Now().Add(5 * time.Second) + for fatHits.Load() == 0 || mockHits.Load() == 0 { + if time.Now().After(deadline) { + t.Fatalf("sources not polled: fat=%d mock=%d", fatHits.Load(), mockHits.Load()) + } + time.Sleep(5 * time.Millisecond) + } + cancel() + + select { + case <-done: + case <-time.After(5 * time.Second): + t.Fatal("Run did not return after cancellation") + } +} diff --git a/internal/fetcher/worker_findATender.go b/internal/fetcher/worker_findATender.go index 5dcf39f..5c1b86b 100644 --- a/internal/fetcher/worker_findATender.go +++ b/internal/fetcher/worker_findATender.go @@ -13,6 +13,8 @@ import ( "git.avsdev.uk/Go/job-search/internal/util" ) +// runFindAOpportunity polls Find a Tender every configured interval until ctx +// is cancelled. func runFindAOpportunity( ctx context.Context, conf *config.JobSearchConfig, since time.Time, ) error { @@ -26,6 +28,9 @@ func runFindAOpportunity( return w.Run(ctx, since) } +// fetchFindAOpportunity fetches everything updated since since from Find a +// Tender. Finding nothing returns ErrNoNewReleases, so the worker still moves +// its checkpoint on. func fetchFindAOpportunity( ctx context.Context, conf *config.JobSearchConfig, since time.Time, ) error { diff --git a/internal/fetcher/worker_ftsMock.go b/internal/fetcher/worker_ftsMock.go index 3d50145..4e04ec5 100644 --- a/internal/fetcher/worker_ftsMock.go +++ b/internal/fetcher/worker_ftsMock.go @@ -13,6 +13,8 @@ import ( "git.avsdev.uk/Go/job-search/internal/util" ) +// runFTSMock polls the mock Find a Tender server every configured interval +// until ctx is cancelled. func runFTSMock( ctx context.Context, conf *config.JobSearchConfig, since time.Time, ) error { @@ -26,6 +28,9 @@ func runFTSMock( return w.Run(ctx, since) } +// fetchFTSMock fetches everything updated since since from the mock Find a +// Tender server. Finding nothing returns ErrNoNewReleases, so the worker still +// moves its checkpoint on. func fetchFTSMock( ctx context.Context, conf *config.JobSearchConfig, since time.Time, ) error { diff --git a/internal/fetcher/worker_sleeper.go b/internal/fetcher/worker_sleeper.go index 3d41391..ce00409 100644 --- a/internal/fetcher/worker_sleeper.go +++ b/internal/fetcher/worker_sleeper.go @@ -30,6 +30,9 @@ func runSleeper( return w.Run(ctx, since) } +// sleep simulates work taking duration, returning early with ctx.Err() if ctx +// is cancelled. With blockShutdown, the work itself ignores ctx (as a +// non-cooperative call would) and only the wait for it stops. func sleep( ctx context.Context, blockShutdown bool, duration time.Duration, ) error { diff --git a/internal/findATender/api_test.go b/internal/findATender/api_test.go new file mode 100644 index 0000000..f908ff4 --- /dev/null +++ b/internal/findATender/api_test.go @@ -0,0 +1,796 @@ +package findATender + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "log/slog" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" + "time" + + "git.avsdev.uk/Go/job-search/internal/config" + "git.avsdev.uk/Go/job-search/internal/findATender/ocds" + resty "resty.dev/v3" +) + +func testConfig(level string) *config.JobSearchConfig { + return &config.JobSearchConfig{Logging: config.JobSearchConfigLogging{Level: level}} +} + +// quietLogs discards slog output for the duration of the test, or sends it +// through h when given. +func quietLogs(t *testing.T, h slog.Handler) { + t.Helper() + if h == nil { + h = slog.NewTextHandler(io.Discard, nil) + } + prev := slog.Default() + slog.SetDefault(slog.New(h)) + t.Cleanup(func() { slog.SetDefault(prev) }) +} + +// newTestFT starts h on a test server and returns a FindATender pointed at it. +func newTestFT(t *testing.T, level string, h http.HandlerFunc) (*FindATender, *httptest.Server) { + t.Helper() + quietLogs(t, nil) + srv := httptest.NewServer(h) + t.Cleanup(srv.Close) + ft := New(testConfig(level), srv.URL) + t.Cleanup(ft.Shutdown) + return ft, srv +} + +func releaseJSON(ocid, id string) string { + return fmt.Sprintf( + `{"ocid":%q,"id":%q,"date":"2026-10-08T12:00:00Z","tag":["tender"],"initiationType":"tender"}`, + ocid, + id, + ) +} + +func packageJSON(next string, releases ...string) string { + links := "" + if next != "" { + links = fmt.Sprintf(`,"links":{"next":%q}`, next) + } + return fmt.Sprintf( + `{"uri":"u","version":"1.1","releases":[%s]%s}`, + strings.Join(releases, ","), + links, + ) +} + +func TestRateSleep(t *testing.T) { + if err := rateSleep(context.Background(), time.Millisecond); err != nil { + t.Errorf("elapsed sleep returned %v", err) + } + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + start := time.Now() + if err := rateSleep(ctx, time.Hour); !errors.Is(err, context.Canceled) { + t.Errorf("cancelled sleep returned %v, want context.Canceled", err) + } + if time.Since(start) > time.Second { + t.Error("cancelled sleep did not return promptly") + } +} + +func TestFetchWithOptsBuildsURI(t *testing.T) { + var gotPath, gotQuery string + ft, _ := newTestFT(t, "warn", func(w http.ResponseWriter, r *http.Request) { + gotPath, gotQuery = r.URL.Path, r.URL.RawQuery + io.WriteString(w, packageJSON("")) + }) + + since := time.Date(2026, 10, 8, 9, 30, 15, 0, time.UTC) + res, err := ft.fetchWithOpts(RequestOptions{Since: since, Limit: 7}) + if err != nil { + t.Fatal(err) + } + res.Body.Close() + if gotPath != "/ocdsReleasePackages" { + t.Errorf("path = %q", gotPath) + } + if gotQuery != "updatedFrom=2026-10-08T09:30:15&limit=7" { + t.Errorf("query = %q", gotQuery) + } + + res, err = ft.fetchWithOpts(RequestOptions{Since: since}) + if err != nil { + t.Fatal(err) + } + res.Body.Close() + if !strings.HasSuffix(gotQuery, "&limit=100") { + t.Errorf("default limit not applied: %q", gotQuery) + } +} + +// A zero Since defaults to 24 hours ago. +func TestFetchWithOptsDefaultSince(t *testing.T) { + var gotQuery string + ft, _ := newTestFT(t, "warn", func(w http.ResponseWriter, r *http.Request) { + gotQuery = r.URL.Query().Get("updatedFrom") + io.WriteString(w, packageJSON("")) + }) + // updatedFrom has whole seconds, so allow for the truncation. + before := time.Now().AddDate(0, 0, -1).Truncate(time.Second) + res, err := ft.fetchWithOpts(RequestOptions{}) + if err != nil { + t.Fatal(err) + } + res.Body.Close() + after := time.Now().AddDate(0, 0, -1) + + from, err := time.ParseInLocation("2006-01-02T15:04:05", gotQuery, time.Local) + if err != nil { + t.Fatalf("updatedFrom = %q: %v", gotQuery, err) + } + if from.Before(before) || from.After(after) { + t.Errorf("updatedFrom = %v, want between %v and %v", from, before, after) + } +} + +func TestFetchURIRateLimit(t *testing.T) { + tests := []struct { + name string + retryAfter string + allowRateLimit bool + wantErr error + wantErrText string + wantHits int32 + }{ + {"not allowed", "0", false, ErrRateLimitExceeded, "", 1}, + {"allowed, retries", "0", true, nil, "", 2}, + {"allowed, bad retry-after", "soon", true, nil, "unknown retry-after value", 1}, + {"allowed, missing retry-after", "", true, nil, "unknown retry-after value", 1}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var hits atomic.Int32 + ft, srv := newTestFT(t, "warn", func(w http.ResponseWriter, r *http.Request) { + if hits.Add(1) == 1 { + if tt.retryAfter != "" { + w.Header().Set("Retry-After", tt.retryAfter) + } + w.WriteHeader(http.StatusTooManyRequests) + return + } + io.WriteString(w, packageJSON("", releaseJSON("ocds-1", "1"))) + }) + + res, err := ft.fetchURI(srv.URL+"/x", tt.allowRateLimit) + switch { + case tt.wantErr != nil: + if !errors.Is(err, tt.wantErr) { + t.Fatalf("err = %v, want %v", err, tt.wantErr) + } + case tt.wantErrText != "": + if err == nil || !strings.Contains(err.Error(), tt.wantErrText) { + t.Fatalf("err = %v, want containing %q", err, tt.wantErrText) + } + default: + if err != nil { + t.Fatal(err) + } + defer res.Body.Close() + if res.StatusCode() != http.StatusOK { + t.Errorf("status = %d after retry", res.StatusCode()) + } + } + if got := hits.Load(); got != tt.wantHits { + t.Errorf("server hit %d times, want %d", got, tt.wantHits) + } + }) + } +} + +func TestFetchURIRateLimitCancelled(t *testing.T) { + ft, srv := newTestFT(t, "warn", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Retry-After", "3600") + w.WriteHeader(http.StatusTooManyRequests) + }) + ctx, cancel := context.WithCancel(context.Background()) + withCtx := ft.With(ctx) + go func() { + time.Sleep(20 * time.Millisecond) + cancel() + }() + + start := time.Now() + if _, err := withCtx.fetchURI(srv.URL, true); !errors.Is(err, ErrFetchCancelled) { + t.Fatalf("err = %v, want ErrFetchCancelled", err) + } + if time.Since(start) > 5*time.Second { + t.Error("cancellation did not interrupt the rate limit wait") + } +} + +func TestFetchURIRateLimitTwice(t *testing.T) { + // A second 429 after waiting is reported as ErrRateLimitExceeded. + var hits atomic.Int32 + ft, srv := newTestFT(t, "warn", func(w http.ResponseWriter, r *http.Request) { + hits.Add(1) + w.Header().Set("Retry-After", "0") + w.WriteHeader(http.StatusTooManyRequests) + }) + if _, err := ft.fetchURI(srv.URL, true); !errors.Is(err, ErrRateLimitExceeded) { + t.Fatalf("err = %v, want ErrRateLimitExceeded", err) + } + if n := hits.Load(); n != 2 { + t.Errorf("requests = %d, want 2", n) + } +} + +func TestFetchURIUnexpectedStatus(t *testing.T) { + for _, status := range []int{ + http.StatusMovedPermanently, http.StatusBadRequest, + http.StatusNotFound, http.StatusInternalServerError, + } { + t.Run(http.StatusText(status), func(t *testing.T) { + ft, srv := newTestFT(t, "warn", func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(status) + io.WriteString(w, packageJSON("", releaseJSON("ocds-1", "1"))) + }) + _, err := ft.fetchURI(srv.URL, true) + if !errors.Is(err, ErrUnexpectedStatus) { + t.Fatalf("err = %v, want ErrUnexpectedStatus", err) + } + if !strings.Contains(err.Error(), fmt.Sprint(status)) { + t.Errorf("err = %q, want it to include the status %d", err, status) + } + }) + } +} + +// A non-2xx after waiting out a 429 is reported as ErrUnexpectedStatus. +func TestFetchURIRateLimitThenError(t *testing.T) { + var hits atomic.Int32 + ft, srv := newTestFT(t, "warn", func(w http.ResponseWriter, r *http.Request) { + if hits.Add(1) == 1 { + w.Header().Set("Retry-After", "0") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusServiceUnavailable) + }) + if _, err := ft.fetchURI(srv.URL, true); !errors.Is(err, ErrUnexpectedStatus) { + t.Fatalf("err = %v, want ErrUnexpectedStatus", err) + } +} + +func TestFetchURIConnectionError(t *testing.T) { + ft, srv := newTestFT(t, "warn", func(w http.ResponseWriter, r *http.Request) {}) + srv.Close() + ft.httpClient.SetRetryCount(0) + if _, err := ft.fetchURI(srv.URL, true); err == nil { + t.Fatal("expected an error from a closed server") + } +} + +func TestFetchURICancelledContext(t *testing.T) { + ft, srv := newTestFT(t, "warn", func(w http.ResponseWriter, r *http.Request) { + io.WriteString(w, packageJSON("")) + }) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if _, err := ft.With(ctx).fetchURI(srv.URL, true); err == nil { + t.Fatal("expected an error from a cancelled context") + } +} + +func TestFetchURIDebugTrace(t *testing.T) { + handlers := map[string]func(io.Writer) slog.Handler{ + "text": func(w io.Writer) slog.Handler { + return slog.NewTextHandler(w, &slog.HandlerOptions{Level: slog.LevelDebug}) + }, + "json": func(w io.Writer) slog.Handler { + return slog.NewJSONHandler(w, &slog.HandlerOptions{Level: slog.LevelDebug}) + }, + } + for name, mk := range handlers { + t.Run(name, func(t *testing.T) { + ft, _ := newTestFT(t, "debug", func(w http.ResponseWriter, r *http.Request) { + io.WriteString(w, packageJSON("", releaseJSON("ocds-1", "1"))) + }) + var logs strings.Builder + quietLogs(t, mk(&logs)) + + // The body must still be readable after the debug logging. + releases, err := ft.fetchOnce(RequestOptions{Since: time.Now()}) + if err != nil { + t.Fatal(err) + } + if len(releases) != 1 { + t.Errorf("got %d releases, want 1", len(releases)) + } + if !strings.Contains(logs.String(), "Resty") { + t.Errorf("no trace logged: %s", logs.String()) + } + }) + } +} + +func TestFetchURIDebugTraceConnectionError(t *testing.T) { + ft, srv := newTestFT(t, "debug", func(w http.ResponseWriter, r *http.Request) {}) + srv.Close() + ft.httpClient.SetRetryCount(0) + if _, err := fetchURI(ft, srv.URL); err == nil { + t.Fatal("expected an error from a closed server") + } +} + +func TestFetchNext(t *testing.T) { + var gotPath string + ft, srv := newTestFT(t, "warn", func(w http.ResponseWriter, r *http.Request) { + gotPath = r.URL.Path + io.WriteString(w, packageJSON("")) + }) + + for name, pkg := range map[string]*ocds.ReleasePackage{ + "nil links": {}, + "empty next": {Links: &ocds.ReleasePackageLinks{Prev: srv.URL}}, + } { + if _, err := ft.fetchNext(pkg, true); !errors.Is(err, ErrNoNext) { + t.Errorf("%s: err = %v, want ErrNoNext", name, err) + } + } + + res, err := ft.fetchNext( + &ocds.ReleasePackage{Links: &ocds.ReleasePackageLinks{Next: srv.URL + "/page2"}}, + true, + ) + if err != nil { + t.Fatal(err) + } + res.Body.Close() + if gotPath != "/page2" { + t.Errorf("fetched %q, want /page2", gotPath) + } +} + +// responseFor fetches body from a test server so it can be unpacked. +func responseFor(t *testing.T, body string) *resty.Response { + t.Helper() + ft, srv := newTestFT(t, "warn", func(w http.ResponseWriter, r *http.Request) { + io.WriteString(w, body) + }) + res, err := ft.fetchURI(srv.URL, false) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { res.Body.Close() }) + return res +} + +func TestSafeUnpackReleases(t *testing.T) { + body := `{ + "uri": "https://example/pkg", + "version": "1.1", + "extensions": ["ext"], + "publishedDate": "2026-10-08T12:00:00Z", + "publisher": {"name": "FTS"}, + "license": "OGL", + "publicationPolicy": "policy", + "links": {"next": "https://example/next"}, + "releases": [ + ` + releaseJSON("ocds-1", "1") + `, + {"ocid": "ocds-bad", "id": "2", "date": 12345}, + {"ocid": "ocds-3", "id": "3", "date": "2026-10-08", "tag": [], "initiationType": "tender"} + ] + }` + pkg, err := safeUnpackReleases(responseFor(t, body)) + if err != nil { + t.Fatal(err) + } + if pkg.URI != "https://example/pkg" || pkg.Version != "1.1" || len(pkg.Extensions) != 1 || + pkg.Publisher == nil || pkg.Publisher.Name != "FTS" || pkg.License != "OGL" || + pkg.PublicationPolicy != "policy" || pkg.Links == nil || pkg.Links.Next != "https://example/next" { + t.Errorf("package fields not copied: %+v", pkg) + } + if pkg.PublishedDate == nil || + !pkg.PublishedDate.Equal(time.Date(2026, 10, 8, 12, 0, 0, 0, time.UTC)) { + t.Errorf("publishedDate = %v", pkg.PublishedDate) + } + if len(pkg.Releases) != 2 || pkg.Releases[0].OCID != "ocds-1" || + pkg.Releases[1].OCID != "ocds-3" { + t.Errorf("releases = %+v, want ocds-1 and ocds-3 (bad one skipped)", pkg.Releases) + } +} + +func TestSafeUnpackReleasesPublishedDate(t *testing.T) { + tests := []struct { + name string + date string // raw JSON value, empty to omit + want *time.Time + }{ + {"omitted", "", nil}, + {"empty string", `""`, nil}, + {"RFC3339", `"2026-10-08T12:00:00Z"`, ptrTime(time.Date(2026, 10, 8, 12, 0, 0, 0, time.UTC))}, + {"date only", `"2026-10-08"`, ptrTime(time.Date(2026, 10, 8, 0, 0, 0, 0, time.UTC))}, + {"unparseable", `"yesterday"`, nil}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + body := `{"releases":[]}` + if tt.date != "" { + body = `{"publishedDate":` + tt.date + `,"releases":[]}` + } + pkg, err := safeUnpackReleases(responseFor(t, body)) + if err != nil { + t.Fatal(err) + } + switch { + case tt.want == nil && pkg.PublishedDate != nil: + t.Errorf("publishedDate = %v, want nil", pkg.PublishedDate) + case tt.want != nil && (pkg.PublishedDate == nil || !pkg.PublishedDate.Equal(*tt.want)): + t.Errorf("publishedDate = %v, want %v", pkg.PublishedDate, tt.want) + } + }) + } +} + +func ptrTime(t time.Time) *time.Time { return &t } + +func TestSafeUnpackReleasesInvalidJSON(t *testing.T) { + for name, body := range map[string]string{ + "not JSON": `oops`, + "wrong shape": `{"releases": {}}`, + "empty body": ``, + "publisher array": `{"publisher": []}`, + } { + t.Run(name, func(t *testing.T) { + if _, err := safeUnpackReleases(responseFor(t, body)); err == nil { + t.Error("expected a decode error") + } + }) + } +} + +// pagedServer serves pages[i] for ?page=i (page 0 at the release package +// path), linking each page to the next. +func pagedServer(t *testing.T, pages ...[]string) (*FindATender, *atomic.Int32) { + t.Helper() + var hits atomic.Int32 + var srvURL string + ft, srv := newTestFT(t, "warn", func(w http.ResponseWriter, r *http.Request) { + hits.Add(1) + i := 0 + fmt.Sscan(r.URL.Query().Get("page"), &i) + next := "" + if i+1 < len(pages) { + next = fmt.Sprintf("%s/ocdsReleasePackages?page=%d", srvURL, i+1) + } + io.WriteString(w, packageJSON(next, pages[i]...)) + }) + srvURL = srv.URL + return ft, &hits +} + +func TestFetchOnce(t *testing.T) { + ft, hits := pagedServer(t, + []string{releaseJSON("ocds-1", "1"), releaseJSON("ocds-2", "1")}, + []string{releaseJSON("ocds-3", "1")}, + ) + releases, err := ft.fetchOnce(RequestOptions{Since: time.Now()}) + if err != nil { + t.Fatal(err) + } + if len(releases) != 2 { + t.Errorf("got %d releases, want 2 (first page only)", len(releases)) + } + if hits.Load() != 1 { + t.Errorf("server hit %d times, want 1", hits.Load()) + } +} + +func TestFetchOnceErrors(t *testing.T) { + ft, _ := pagedServer(t, []string{}) + if _, err := ft.fetchOnce(RequestOptions{Since: time.Now()}); !errors.Is(err, ErrNoReleases) { + t.Errorf("empty page: err = %v, want ErrNoReleases", err) + } + + ft, _ = newTestFT(t, "warn", func(w http.ResponseWriter, r *http.Request) { + io.WriteString(w, "not json") + }) + if _, err := ft.fetchOnce(RequestOptions{Since: time.Now()}); err == nil { + t.Error("bad body: expected an error") + } + + ft, _ = newTestFT(t, "warn", func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusTooManyRequests) + }) + if _, err := ft.fetchOnce( + RequestOptions{Since: time.Now()}, + ); !errors.Is( + err, + ErrRateLimitExceeded, + ) { + t.Errorf("rate limited: err = %v, want ErrRateLimitExceeded", err) + } +} + +func TestFetchAll(t *testing.T) { + ft, hits := pagedServer(t, + []string{releaseJSON("ocds-1", "1"), releaseJSON("ocds-2", "1")}, + []string{}, + []string{releaseJSON("ocds-1", "2")}, + ) + releases, err := ft.fetchAll(RequestOptions{Since: time.Now()}) + if err != nil { + t.Fatal(err) + } + var ids []string + for _, r := range releases { + ids = append(ids, r.OCID+"/"+r.ID) + } + if strings.Join(ids, ",") != "ocds-1/1,ocds-2/1,ocds-1/2" { + t.Errorf("releases = %v", ids) + } + if hits.Load() != 3 { + t.Errorf("server hit %d times, want 3", hits.Load()) + } +} + +func TestFetchAllErrors(t *testing.T) { + ft, _ := pagedServer(t, []string{}, []string{}) + if _, err := ft.fetchAll(RequestOptions{Since: time.Now()}); !errors.Is(err, ErrNoReleases) { + t.Errorf("all pages empty: err = %v, want ErrNoReleases", err) + } + + // First page fails. + ft, _ = newTestFT(t, "warn", func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusTooManyRequests) + }) + if _, err := ft.fetchAll( + RequestOptions{Since: time.Now()}, + ); !errors.Is( + err, + ErrRateLimitExceeded, + ) { + t.Errorf("first page rate limited: err = %v", err) + } + + ft, _ = newTestFT(t, "warn", func(w http.ResponseWriter, r *http.Request) { + io.WriteString(w, "not json") + }) + if _, err := ft.fetchAll(RequestOptions{Since: time.Now()}); err == nil { + t.Error("first page bad body: expected an error") + } + + // A later page fails, either to fetch or to decode. + for name, fail := range map[string]http.HandlerFunc{ + "rate limited": func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusTooManyRequests) }, + "bad body": func(w http.ResponseWriter, r *http.Request) { io.WriteString(w, "not json") }, + } { + t.Run(name, func(t *testing.T) { + var srvURL string + ft, srv := newTestFT(t, "warn", func(w http.ResponseWriter, r *http.Request) { + if r.URL.Query().Get("page") == "" { + io.WriteString(w, packageJSON(srvURL+"/next?page=1", releaseJSON("ocds-1", "1"))) + return + } + fail(w, r) + }) + srvURL = srv.URL + if _, err := ft.fetchAll(RequestOptions{Since: time.Now()}); err == nil { + t.Error("expected an error from the second page") + } + }) + } +} + +// fetchAll stops, keeping what it has, when a next link repeats an earlier one. +func TestFetchAllRepeatedNextLink(t *testing.T) { + tests := map[string]func(base string, page string) string{ + // page 0 -> page 1 -> page 1 ... + "self link": func(base, page string) string { return base + "/next?page=1" }, + // page 0 -> page 1 -> page 2 -> page 1 ... + "cycle": func(base, page string) string { + if page == "1" { + return base + "/next?page=2" + } + return base + "/next?page=1" + }, + } + for name, nextFor := range tests { + t.Run(name, func(t *testing.T) { + var hits atomic.Int32 + var srvURL string + ft, srv := newTestFT(t, "error", func(w http.ResponseWriter, r *http.Request) { + if hits.Add(1) > 10 { + t.Error("fetchAll did not stop following next links") + w.WriteHeader(http.StatusBadRequest) + return + } + page := r.URL.Query().Get("page") + io.WriteString(w, packageJSON( + nextFor(srvURL, page), releaseJSON("ocds-"+page, "1"), + )) + }) + srvURL = srv.URL + + releases, err := ft.fetchAll(RequestOptions{Since: time.Now()}) + if err != nil { + t.Fatal(err) + } + if len(releases) != int(hits.Load()) { + t.Errorf("releases = %d, want one per request (%d)", len(releases), hits.Load()) + } + }) + } +} + +func TestUnmarshalLenientDates(t *testing.T) { + raw := json.RawMessage(`{ + "date": "2006-10-02", + "publishedDate": "", + "tender": { + "tenderPeriod": {"startDate": "2006-10-02T15:33:02+07:00", "endDate": "2006-11-30"}, + "description": "Bids close on 2006-11-30", + "expressionOfInterestDeadline": "2006-10-20" + }, + "awards": [{"date": "2006-12-01", "id": "2006-12-01"}], + "value": 1234567.891 + }`) + + var got struct { + Date time.Time `json:"date"` + PublishedDate *time.Time `json:"publishedDate"` + Tender struct { + TenderPeriod struct { + StartDate *time.Time `json:"startDate"` + EndDate *time.Time `json:"endDate"` + } `json:"tenderPeriod"` + Description string `json:"description"` + ExpressionOfInterestDeadline *time.Time `json:"expressionOfInterestDeadline"` + } `json:"tender"` + Awards []struct { + Date *time.Time `json:"date"` + ID string `json:"id"` + } `json:"awards"` + Value json.Number `json:"value"` + } + if err := unmarshalLenientDates(raw, &got); err != nil { + t.Fatal(err) + } + + midnight := func(y int, m time.Month, d int) time.Time { return time.Date(y, m, d, 0, 0, 0, 0, time.UTC) } + if !got.Date.Equal(midnight(2006, 10, 2)) { + t.Errorf("date = %v", got.Date) + } + if got.PublishedDate != nil { + t.Errorf("empty publishedDate should decode as nil, got %v", got.PublishedDate) + } + if start := got.Tender.TenderPeriod.StartDate; start == nil || + !start.Equal(time.Date(2006, 10, 2, 15, 33, 2, 0, time.FixedZone("", 7*3600))) { + t.Errorf("full timestamp changed: %v", start) + } + if end := got.Tender.TenderPeriod.EndDate; end == nil || !end.Equal(midnight(2006, 11, 30)) { + t.Errorf("endDate = %v", end) + } + if d := got.Tender.ExpressionOfInterestDeadline; d == nil || !d.Equal(midnight(2006, 10, 20)) { + t.Errorf("expressionOfInterestDeadline = %v", d) + } + if got.Tender.Description != "Bids close on 2006-11-30" { + t.Errorf("free text changed: %q", got.Tender.Description) + } + if a := got.Awards[0]; a.Date == nil || !a.Date.Equal(midnight(2006, 12, 1)) || + a.ID != "2006-12-01" { + t.Errorf("award = %+v (date should be repaired, id left alone)", a) + } + if got.Value != "1234567.891" { + t.Errorf("number changed: %s", got.Value) + } +} + +func TestRepairDatesLeavesValidJSONAlone(t *testing.T) { + raw := json.RawMessage(`{"date": "2006-10-02T15:33:02+07:00", "title": "x"}`) + out, err := repairDates(raw) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(out, raw) { + t.Errorf("valid input was rewritten: %s", out) + } +} + +func TestRepairDatesRejectsBadJSON(t *testing.T) { + for _, raw := range []string{`{`, `{"date": "2006-10-02"} trailing`} { + if _, err := repairDates(json.RawMessage(raw)); err == nil { + t.Errorf("expected an error for %q", raw) + } + } +} + +func TestUnmarshalLenientDatesErrors(t *testing.T) { + var v struct { + Date time.Time `json:"date"` + } + tests := map[string]string{ + "bad JSON": `{`, + "wrong type": `{"date": 5}`, + "unparseable date": `{"date": "02/10/2006"}`, + "not a JSON object": `"2006-10-02"`, + } + for name, raw := range tests { + t.Run(name, func(t *testing.T) { + if err := unmarshalLenientDates(json.RawMessage(raw), &v); err == nil { + t.Errorf("expected an error for %s", raw) + } + }) + } +} + +func TestReleaseIDs(t *testing.T) { + tests := []struct { + name string + raw string + wantOCID string + wantID string + wantErr string // substring; empty means no error + }{ + {"ocid then id", `{"ocid":"ocds-1","id":"r1"}`, "ocds-1", "r1", ""}, + {"id then ocid", `{"id":"r1","ocid":"ocds-1"}`, "ocds-1", "r1", ""}, + { + "numeric id keeps its digits", + `{"ocid":"ocds-1","id":12345678901234567890}`, + "ocds-1", + "12345678901234567890", + "", + }, + { + "skips other fields", + `{"date":"2026-01-01","tender":{"items":[1,2]},"ocid":"ocds-1","id":"r1"}`, + "ocds-1", + "r1", + "", + }, + { + "ignores errors after both ids", + `{"ocid":"ocds-1","id":"r1","tender":{"value":"oops"},` + `"bad":}`, + "ocds-1", + "r1", + "", + }, + {"empty input", ``, "", "", "reading release"}, + {"not an object", `["ocid","id"]`, "", "", "not a JSON object"}, + {"empty object", `{}`, "", "", "no ocid or id"}, + {"missing ocid", `{"id":"r1"}`, "", "r1", "no ocid"}, + {"missing id", `{"ocid":"ocds-1"}`, "ocds-1", "", "no id"}, + {"ocid wrong type", `{"ocid":5,"id":"r1"}`, "", "", "reading release ocid"}, + {"id wrong type", `{"ocid":"ocds-1","id":true}`, "ocds-1", "", "unexpected bool"}, + {"id is an object", `{"ocid":"ocds-1","id":{}}`, "ocds-1", "", "unexpected map"}, + {"invalid id JSON", `{"ocid":"ocds-1","id":}`, "ocds-1", "", "reading release id"}, + { + "invalid field before ids", + `{"date":nope,"ocid":"ocds-1","id":"r1"}`, + "", + "", + `reading release field "date"`, + }, + {"truncated after ocid", `{"ocid":"ocds-1",`, "ocds-1", "", "reading release"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ocid, id, err := releaseIDs(json.RawMessage(tt.raw)) + if tt.wantErr == "" { + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + } else if err == nil || !strings.Contains(err.Error(), tt.wantErr) { + t.Fatalf("err = %v, want containing %q", err, tt.wantErr) + } + if ocid != tt.wantOCID || id != tt.wantID { + t.Errorf("got (%q, %q), want (%q, %q)", ocid, id, tt.wantOCID, tt.wantID) + } + }) + } +} diff --git a/internal/findATender/findATender.go b/internal/findATender/findATender.go index 84688cf..eb66897 100644 --- a/internal/findATender/findATender.go +++ b/internal/findATender/findATender.go @@ -16,6 +16,8 @@ type FindATender struct { ctx context.Context } +// New returns a client for the OCDS API at apiRoot. Requests time out after +// 60 seconds and are retried once. Call Shutdown when finished with it. func New(conf *config.JobSearchConfig, apiRoot string) *FindATender { ft := &FindATender{ config: conf, @@ -27,10 +29,14 @@ func New(conf *config.JobSearchConfig, apiRoot string) *FindATender { return ft } +// ReloadConfig replaces the configuration the client reads, such as the log +// level. The HTTP client and API root set by New are not changed. func (ft *FindATender) ReloadConfig(conf *config.JobSearchConfig) { ft.config = conf } +// With returns a copy of the client whose requests use ctx, so they stop +// when it is cancelled. A nil ctx returns the client unchanged. func (ft *FindATender) With(ctx context.Context) *FindATender { if ctx == nil { return ft @@ -41,10 +47,14 @@ func (ft *FindATender) With(ctx context.Context) *FindATender { return &copied } +// Shutdown closes the client's HTTP connections. func (ft *FindATender) Shutdown() { ft.httpClient.Close() } +// FetchOnce fetches the first page of releases updated since since, and +// converts them to opportunities. If allowRateLimit is true, a 429 response +// is waited out and retried; otherwise it fails with ErrRateLimitExceeded. func (ft *FindATender) FetchOnce( since time.Time, allowRateLimit bool, @@ -59,6 +69,9 @@ func (ft *FindATender) FetchOnce( return Convert(releases), nil } +// FetchAll fetches every page of releases updated since since, following +// each page's next link, and converts them to opportunities. allowRateLimit +// behaves as for FetchOnce. func (ft *FindATender) FetchAll( since time.Time, allowRateLimit bool, diff --git a/internal/findATender/findATender_test.go b/internal/findATender/findATender_test.go new file mode 100644 index 0000000..d2442af --- /dev/null +++ b/internal/findATender/findATender_test.go @@ -0,0 +1,124 @@ +package findATender + +import ( + "context" + "errors" + "io" + "net/http" + "slices" + "strings" + "testing" + "time" + + "git.avsdev.uk/Go/job-search/internal/types" +) + +func TestNew(t *testing.T) { + conf := testConfig("warn") + ft := New(conf, "https://example/api") + defer ft.Shutdown() + + if ft.config != conf || ft.apiRoot != "https://example/api" { + t.Errorf("New did not keep its arguments: %+v", ft) + } + if ft.httpClient == nil { + t.Fatal("no http client") + } + if ft.httpClient.Timeout() != 60*time.Second || ft.httpClient.RetryCount() != 1 { + t.Errorf("client timeout %v, retries %d", ft.httpClient.Timeout(), ft.httpClient.RetryCount()) + } + if ft.ctx != nil { + t.Error("New should not set a context") + } +} + +func TestReloadConfig(t *testing.T) { + ft := New(testConfig("warn"), "") + defer ft.Shutdown() + conf := testConfig("debug") + ft.ReloadConfig(conf) + if ft.config != conf { + t.Error("config not replaced") + } +} + +func TestWith(t *testing.T) { + ft := New(testConfig("warn"), "") + defer ft.Shutdown() + + // Passing nil on purpose to test that case. + //nolint:staticcheck // SA1012 + if got := ft.With(nil); got != ft { + t.Error("With(nil) should return the receiver") + } + + type key struct{} + ctx := context.WithValue(context.Background(), key{}, "v") + got := ft.With(ctx) + if got == ft { + t.Fatal("With should return a copy") + } + if got.ctx != ctx { + t.Error("copy does not carry the context") + } + if ft.ctx != nil { + t.Error("With changed the original") + } + if got.httpClient != ft.httpClient || got.config != ft.config { + t.Error("copy should share the client and config") + } +} + +func opportunityIDs(ops []*types.Opportunity) []string { + var ids []string + for _, op := range ops { + ids = append(ids, op.UUID) + } + slices.Sort(ids) + return ids +} + +func TestFetchOncePublic(t *testing.T) { + ft, _ := pagedServer(t, + []string{releaseJSON("ocds-1", "1"), releaseJSON("ocds-1", "2"), releaseJSON("ocds-2", "1")}, + []string{releaseJSON("ocds-3", "1")}, + ) + ops, err := ft.FetchOnce(time.Now(), false) + if err != nil { + t.Fatal(err) + } + if got := strings.Join(opportunityIDs(ops), ","); got != "ocds-1,ocds-2" { + t.Errorf("opportunities = %s", got) + } + + ft, _ = pagedServer(t, []string{}) + if _, err := ft.FetchOnce(time.Now(), false); !errors.Is(err, ErrNoReleases) { + t.Errorf("err = %v, want ErrNoReleases", err) + } +} + +func TestFetchAllPublic(t *testing.T) { + ft, _ := pagedServer(t, + []string{releaseJSON("ocds-1", "1"), releaseJSON("ocds-2", "1")}, + []string{releaseJSON("ocds-1", "2"), releaseJSON("ocds-3", "1")}, + ) + ops, err := ft.FetchAll(time.Now(), false) + if err != nil { + t.Fatal(err) + } + if got := strings.Join(opportunityIDs(ops), ","); got != "ocds-1,ocds-2,ocds-3" { + t.Errorf("opportunities = %s", got) + } + for _, op := range ops { + if op.UUID == "ocds-1" && len(op.Releases) != 2 { + t.Errorf("ocds-1 has %d releases, want 2", len(op.Releases)) + } + } + + ft, _ = newTestFT(t, "warn", func(w http.ResponseWriter, r *http.Request) { + io.WriteString(w, "not json") + }) + if _, err := ft.FetchAll(time.Now(), false); err == nil { + t.Error("expected an error") + } +} diff --git a/internal/findATender/typeConvert.go b/internal/findATender/typeConvert.go index 3598cd0..782ab41 100644 --- a/internal/findATender/typeConvert.go +++ b/internal/findATender/typeConvert.go @@ -9,12 +9,16 @@ import ( "git.avsdev.uk/Go/job-search/internal/types" ) +// ocdsGetBuyerId returns the buyer's id, or "" if the release has no buyer. func ocdsGetBuyerId(release *ocds.Release) string { if release.Buyer != nil { return string(release.Buyer.ID) } return "" } + +// ocdsGetBuyerName returns the buyer's name, or "" if the release has no +// buyer. func ocdsGetBuyerName(release *ocds.Release) string { if release.Buyer != nil { return release.Buyer.Name @@ -22,6 +26,8 @@ func ocdsGetBuyerName(release *ocds.Release) string { return "" } +// ocdsConvertClassification converts an OCDS classification, noting whether +// it is a CPV code. func ocdsConvertClassification( classif *ocds.Classification, ) types.ClassificationCode { @@ -32,6 +38,8 @@ func ocdsConvertClassification( } } +// ocdsGetClassifications returns the tender's classification, then each +// item's main and additional classifications, without repeating a code. func ocdsGetClassifications(release *ocds.Release) []types.ClassificationCode { classifications := make([]types.ClassificationCode, 0) if release.Tender == nil { @@ -62,6 +70,8 @@ func ocdsGetClassifications(release *ocds.Release) []types.ClassificationCode { return classifications } +// ocdsIsSMESuitable reports whether any lot of the tender is marked as +// suitable for SMEs. A tender with no lots is not. func ocdsIsSMESuitable(release *ocds.Release) bool { if release.Tender == nil { return false @@ -78,6 +88,10 @@ func ocdsIsSMESuitable(release *ocds.Release) bool { } return false } + +// ocdsIsVCSESuitable reports whether any lot of the tender is marked as +// suitable for VCSEs (voluntary, community and social enterprises). A tender +// with no lots is not. func ocdsIsVCSESuitable(release *ocds.Release) bool { if release.Tender == nil { return false @@ -95,6 +109,8 @@ func ocdsIsVCSESuitable(release *ocds.Release) bool { return false } +// ocdsRelease converts a release's identity, tags and date, and keeps a +// re-encoded copy of the whole release as JSON. func ocdsRelease(release *ocds.Release) types.OpportunityRelease { tags := make([]types.OpportunityReleaseTag, 0) for _, val := range release.Tag { @@ -120,6 +136,8 @@ func ocdsRelease(release *ocds.Release) types.OpportunityRelease { return rel } +// convertOne builds an opportunity from a single release. Convert passes it +// each ocid's latest release, as that has the current state of the tender. func convertOne(release *ocds.Release) *types.Opportunity { t := &types.Opportunity{ UUID: release.OCID, diff --git a/internal/findATender/typeConvert_test.go b/internal/findATender/typeConvert_test.go new file mode 100644 index 0000000..72c214c --- /dev/null +++ b/internal/findATender/typeConvert_test.go @@ -0,0 +1,330 @@ +package findATender + +import ( + "encoding/json" + "reflect" + "slices" + "testing" + "time" + + "git.avsdev.uk/Go/job-search/internal/findATender/ocds" + "git.avsdev.uk/Go/job-search/internal/types" +) + +func boolPtr(b bool) *bool { return &b } + +func lotWith(s *ocds.Suitability) ocds.Lot { return ocds.Lot{ID: "1", Suitability: s} } + +func TestOcdsGetBuyer(t *testing.T) { + tests := []struct { + name string + buyer *ocds.OrganizationReference + wantID, wantName string + }{ + {"nil buyer", nil, "", ""}, + {"empty buyer", &ocds.OrganizationReference{}, "", ""}, + {"full buyer", &ocds.OrganizationReference{ID: "GB-1", Name: "Council"}, "GB-1", "Council"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + r := &ocds.Release{Buyer: tt.buyer} + if got := ocdsGetBuyerId(r); got != tt.wantID { + t.Errorf("buyer id = %q, want %q", got, tt.wantID) + } + if got := ocdsGetBuyerName(r); got != tt.wantName { + t.Errorf("buyer name = %q, want %q", got, tt.wantName) + } + }) + } +} + +func TestOcdsConvertClassification(t *testing.T) { + tests := []struct { + name string + in ocds.Classification + want types.ClassificationCode + }{ + {"CPV", ocds.Classification{Scheme: "CPV", ID: "72000000", Description: "IT services"}, + types.ClassificationCode{IsCPV: true, Code: "72000000", Classification: "IT services"}}, + {"non-CPV", ocds.Classification{Scheme: "UNSPSC", ID: "43000000", Description: "IT"}, + types.ClassificationCode{IsCPV: false, Code: "43000000", Classification: "IT"}}, + {"scheme is case sensitive", ocds.Classification{Scheme: "cpv", ID: "1"}, + types.ClassificationCode{IsCPV: false, Code: "1"}}, + {"empty", ocds.Classification{}, types.ClassificationCode{}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := ocdsConvertClassification(&tt.in); got != tt.want { + t.Errorf("got %+v, want %+v", got, tt.want) + } + }) + } +} + +func TestOcdsGetClassifications(t *testing.T) { + cpv := func(id string) ocds.Classification { + return ocds.Classification{Scheme: "CPV", ID: ocds.ID(id), Description: "d" + id} + } + codes := func(cs []types.ClassificationCode) []string { + out := []string{} + for _, c := range cs { + out = append(out, c.Code) + } + return out + } + main := cpv("1") + tests := []struct { + name string + tender *ocds.Tender + want []string + }{ + {"nil tender", nil, []string{}}, + {"no classifications", &ocds.Tender{}, []string{}}, + {"main only", &ocds.Tender{Classification: &main}, []string{"1"}}, + {"main and item additionals, deduplicated", &ocds.Tender{ + Classification: &main, + Items: []ocds.Item{ + {ID: "a", AdditionalClassifications: []ocds.Classification{cpv("2"), cpv("1")}}, + {ID: "b"}, + {ID: "c", AdditionalClassifications: []ocds.Classification{cpv("3"), cpv("2")}}, + }, + }, []string{"1", "2", "3"}}, + {"item additionals without main", &ocds.Tender{ + Items: []ocds.Item{{ID: "a", AdditionalClassifications: []ocds.Classification{cpv("5")}}}, + }, []string{"5"}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := ocdsGetClassifications(&ocds.Release{Tender: tt.tender}) + if got == nil { + t.Fatal("got nil slice, want non-nil") + } + if !slices.Equal(codes(got), tt.want) { + t.Errorf("codes = %v, want %v", codes(got), tt.want) + } + }) + } +} + +func TestOcdsGetClassificationsIncludesItemClassification(t *testing.T) { + item := ocds.Classification{Scheme: "CPV", ID: "9"} + got := ocdsGetClassifications(&ocds.Release{Tender: &ocds.Tender{ + Items: []ocds.Item{{ID: "a", Classification: &item}}, + }}) + if len(got) != 1 || got[0].Code != "9" { + t.Errorf("got %+v, want the item classification 9", got) + } +} + +func TestOcdsSuitability(t *testing.T) { + tests := []struct { + name string + tender *ocds.Tender + wantSME bool + wantVCSE bool + }{ + {"nil tender", nil, false, false}, + {"no lots", &ocds.Tender{}, false, false}, + {"lot without suitability", &ocds.Tender{Lots: []ocds.Lot{lotWith(nil)}}, false, false}, + { + "suitability fields nil", + &ocds.Tender{Lots: []ocds.Lot{lotWith(&ocds.Suitability{})}}, + false, + false, + }, + {"explicitly false", &ocds.Tender{Lots: []ocds.Lot{ + lotWith(&ocds.Suitability{SME: boolPtr(false), Vcse: boolPtr(false)}), + }}, false, false}, + {"SME only", &ocds.Tender{Lots: []ocds.Lot{ + lotWith(&ocds.Suitability{SME: boolPtr(true)}), + }}, true, false}, + {"VCSE only", &ocds.Tender{Lots: []ocds.Lot{ + lotWith(&ocds.Suitability{Vcse: boolPtr(true)}), + }}, false, true}, + {"any lot suffices", &ocds.Tender{Lots: []ocds.Lot{ + lotWith(nil), + lotWith(&ocds.Suitability{SME: boolPtr(false)}), + lotWith(&ocds.Suitability{SME: boolPtr(true), Vcse: boolPtr(true)}), + }}, true, true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + r := &ocds.Release{Tender: tt.tender} + if got := ocdsIsSMESuitable(r); got != tt.wantSME { + t.Errorf("SME = %v, want %v", got, tt.wantSME) + } + if got := ocdsIsVCSESuitable(r); got != tt.wantVCSE { + t.Errorf("VCSE = %v, want %v", got, tt.wantVCSE) + } + }) + } +} + +func TestOcdsRelease(t *testing.T) { + date := time.Date(2026, 10, 8, 12, 0, 0, 0, time.UTC) + r := &ocds.Release{OCID: "ocds-x-1", ID: "001", Date: date, Tag: []string{"planning", "tender"}} + + got := ocdsRelease(r) + if got.UUID != "ocds-x-1" || got.ReleaseID != "001" || !got.ReleaseDate.Equal(date) { + t.Errorf("release = %+v", got) + } + want := []types.OpportunityReleaseTag{ + {UUID: "ocds-x-1", ReleaseID: "001", Tag: "planning"}, + {UUID: "ocds-x-1", ReleaseID: "001", Tag: "tender"}, + } + if !reflect.DeepEqual(got.Tags, want) { + t.Errorf("tags = %+v, want %+v", got.Tags, want) + } + + var back ocds.Release + if err := json.Unmarshal([]byte(got.JSON), &back); err != nil { + t.Fatalf("JSON does not round trip: %v (%s)", err, got.JSON) + } + if back.OCID != r.OCID || back.ID != r.ID || !back.Date.Equal(date) { + t.Errorf("round tripped release = %+v", back) + } + + if empty := ocdsRelease(&ocds.Release{}); empty.Tags == nil || len(empty.Tags) != 0 { + t.Errorf("no tags should give an empty, non-nil slice, got %#v", empty.Tags) + } +} + +func TestConvertOne(t *testing.T) { + start := time.Date(2026, 10, 1, 0, 0, 0, 0, time.UTC) + end := time.Date(2026, 11, 1, 0, 0, 0, 0, time.UTC) + amount := 125000.5 + main := ocds.Classification{Scheme: "CPV", ID: "72000000", Description: "IT"} + r := &ocds.Release{ + OCID: "ocds-x-1", + ID: "001", + Tag: []string{"tender"}, + Buyer: &ocds.OrganizationReference{ID: "GB-1", Name: "Council"}, + Tender: &ocds.Tender{ + Title: "Website", + Status: "active", + Description: "Build a website", + Classification: &main, + MainProcurementCategory: "services", + ProcurementMethod: "open", + Lots: []ocds.Lot{lotWith(&ocds.Suitability{SME: boolPtr(true)})}, + TenderPeriod: &ocds.Period{StartDate: &start, EndDate: &end}, + Value: &ocds.Value{Amount: &amount}, + Documents: []ocds.Document{{ID: "doc"}}, + }, + } + + got := convertOne(r) + checks := []struct { + field string + got, want any + }{ + {"UUID", got.UUID, "ocds-x-1"}, + {"UUIDType", got.UUIDType, "ocid"}, + {"BuyerId", got.BuyerId, "GB-1"}, + {"BuyerName", got.BuyerName, "Council"}, + {"Title", got.Title, "Website"}, + {"Status", got.Status, "active"}, + {"Description", got.Description, "Build a website"}, + {"MainCategory", got.MainCategory, "services"}, + {"ProcurementMethod", got.ProcurementMethod, "open"}, + {"SMESuitable", got.SMESuitable, true}, + {"VCSESuitable", got.VCSESuitable, false}, + {"StartDate", got.StartDate, &start}, + {"EndDate", got.EndDate, &end}, + {"GrossValue", got.GrossValue, 125000.5}, + {"HasDocuments", got.HasDocuments, true}, + {"Classifications", len(got.Classifications), 1}, + {"Releases", len(got.Releases), 1}, + } + for _, c := range checks { + if !reflect.DeepEqual(c.got, c.want) { + t.Errorf("%s = %v, want %v", c.field, c.got, c.want) + } + } +} + +func TestConvertOneMinimal(t *testing.T) { + tests := []struct { + name string + tender *ocds.Tender + }{ + {"nil tender", nil}, + {"empty tender", &ocds.Tender{}}, + {"value without amount", &ocds.Tender{Value: &ocds.Value{}}}, + {"empty tender period", &ocds.Tender{TenderPeriod: &ocds.Period{}}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := convertOne(&ocds.Release{OCID: "ocds-x-2", ID: "1", Tender: tt.tender}) + if got.UUID != "ocds-x-2" || got.BuyerId != "" || got.BuyerName != "" { + t.Errorf("got %+v", got) + } + if got.GrossValue != 0 || got.StartDate != nil || got.EndDate != nil || got.HasDocuments { + t.Errorf("unexpected tender fields: %+v", got) + } + if len(got.Releases) != 1 { + t.Errorf("releases = %d, want 1", len(got.Releases)) + } + }) + } +} + +func TestConvert(t *testing.T) { + if got := Convert(nil); len(got) != 0 { + t.Errorf("Convert(nil) = %v, want empty", got) + } + + day := func(d int) time.Time { return time.Date(2026, 1, d, 0, 0, 0, 0, time.UTC) } + releases := []ocds.Release{ + {OCID: "ocds-b", ID: "1", Date: day(1), Tender: &ocds.Tender{Title: "B"}}, + {OCID: "ocds-a", ID: "2", Date: day(3), Tender: &ocds.Tender{ + Title: "A latest", Lots: []ocds.Lot{{ID: "1"}, {ID: "2"}}, + }}, + {OCID: "ocds-a", ID: "1", Date: day(2), Tender: &ocds.Tender{Title: "A first"}}, + {OCID: "ocds-a", ID: "3", Date: day(1), Tender: &ocds.Tender{Title: "A oldest"}}, + } + got := Convert(releases) + + // Opportunities come out in the order each ocid is first seen. + if len(got) != 2 { + t.Fatalf("got %d opportunities, want 2", len(got)) + } + b, a := got[0], got[1] + if a.UUID != "ocds-a" || b.UUID != "ocds-b" { + t.Fatalf("uuids = %s, %s; want ocds-b, ocds-a", b.UUID, a.UUID) + } + + // Fields come from the latest release; releases are listed oldest first. + if a.Title != "A latest" { + t.Errorf("ocds-a title = %q, want %q", a.Title, "A latest") + } + if !a.LatestReleaseDate.Equal(day(3)) { + t.Errorf("ocds-a LatestReleaseDate = %v, want %v", a.LatestReleaseDate, day(3)) + } + if !a.MultiLot { + t.Error("ocds-a MultiLot = false, want true for two lots") + } + var ids []string + for _, r := range a.Releases { + ids = append(ids, r.ReleaseID) + } + if want := []string{"3", "1", "2"}; !slices.Equal(ids, want) { + t.Errorf("ocds-a release ids = %v, want %v", ids, want) + } + + if b.Title != "B" || b.MultiLot || !b.LatestReleaseDate.Equal(day(1)) || len(b.Releases) != 1 { + t.Errorf("ocds-b = %+v", b) + } +} + +// Releases with the same date keep their input order, so the later one wins. +func TestConvertSameDate(t *testing.T) { + date := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + got := Convert([]ocds.Release{ + {OCID: "ocds-a", ID: "1", Date: date, Tender: &ocds.Tender{Title: "first"}}, + {OCID: "ocds-a", ID: "2", Date: date, Tender: &ocds.Tender{Title: "second"}}, + }) + if len(got) != 1 || got[0].Title != "second" { + t.Fatalf("got %+v, want one opportunity titled %q", got, "second") + } +} diff --git a/internal/logging/accesslog.go b/internal/logging/accesslog.go index 15e2b59..f12b4d9 100644 --- a/internal/logging/accesslog.go +++ b/internal/logging/accesslog.go @@ -65,6 +65,9 @@ func clientKey(req *http.Request) string { return host } +// AccessLog wraps next, writing a line to logger for each request in the +// Apache/nginx Common Log Format, or the Combined Log Format (which adds the +// referer and user agent) if combined is true. func AccessLog(logger *log.Logger, combined bool, next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { start := time.Now() diff --git a/internal/logging/accesslog_test.go b/internal/logging/accesslog_test.go new file mode 100644 index 0000000..0a1f668 --- /dev/null +++ b/internal/logging/accesslog_test.go @@ -0,0 +1,138 @@ +package logging + +import ( + "bytes" + "log" + "net/http" + "net/http/httptest" + "regexp" + "strings" + "testing" +) + +func TestLogField(t *testing.T) { + tests := []struct { + in, want string + }{ + {"", "-"}, + {"plain", "plain"}, + {"GET /a?b=c HTTP/1.1", "GET /a?b=c HTTP/1.1"}, + {`say "hi"`, `say \x22hi\x22`}, + {`back\slash`, `back\x5Cslash`}, + {"tab\tnl\n", `tab\x09nl\x0A`}, + {"del\x7f", `del\x7F`}, + {"héllo", "héllo"}, // multi-byte UTF-8 passes through + } + for _, tc := range tests { + if got := logField(tc.in); got != tc.want { + t.Errorf("logField(%q) = %q, want %q", tc.in, got, tc.want) + } + } +} + +func TestClientKey(t *testing.T) { + tests := []struct { + remote, want string + }{ + {"192.0.2.1:1234", "192.0.2.1"}, + {"[2001:db8::1]:443", "2001:db8::1"}, + {"no-port", "no-port"}, + {"", ""}, + } + for _, tc := range tests { + req := &http.Request{RemoteAddr: tc.remote} + if got := clientKey(req); got != tc.want { + t.Errorf("clientKey(%q) = %q, want %q", tc.remote, got, tc.want) + } + } +} + +func TestStatusRecorder(t *testing.T) { + t.Run("first WriteHeader wins", func(t *testing.T) { + rec := &statusRecorder{ResponseWriter: httptest.NewRecorder()} + rec.WriteHeader(http.StatusTeapot) + rec.WriteHeader(http.StatusInternalServerError) + if rec.status != http.StatusTeapot { + t.Fatalf("status = %d", rec.status) + } + }) + t.Run("Write implies 200 and counts bytes", func(t *testing.T) { + inner := httptest.NewRecorder() + rec := &statusRecorder{ResponseWriter: inner} + rec.Write([]byte("hello")) + rec.Write([]byte(" world")) + if rec.status != http.StatusOK || rec.bytes != 11 { + t.Fatalf("status = %d, bytes = %d", rec.status, rec.bytes) + } + if inner.Body.String() != "hello world" { + t.Fatalf("body = %q", inner.Body.String()) + } + }) +} + +var clfTime = `\[\d{2}/[A-Z][a-z]{2}/\d{4}:\d{2}:\d{2}:\d{2} [+-]\d{4}\]` + +func TestAccessLog(t *testing.T) { + tests := []struct { + name string + combined bool + handler http.HandlerFunc + setup func(*http.Request) + want string // regexp for the whole line + }{ + { + name: "no body defaults to 200 and -", + handler: func(http.ResponseWriter, *http.Request) {}, + want: `^192\.0\.2\.1 - - ` + clfTime + ` "GET /path\?q=1 HTTP/1\.1" 200 -$`, + }, + { + name: "status and size", + handler: func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusNotFound) + w.Write([]byte("nope")) + }, + want: `^192\.0\.2\.1 - - ` + clfTime + ` "GET /path\?q=1 HTTP/1\.1" 404 4$`, + }, + { + name: "basic auth user is escaped", + handler: func(w http.ResponseWriter, _ *http.Request) { w.Write([]byte("x")) }, + setup: func(r *http.Request) { r.SetBasicAuth(`bo"b`, "pw") }, + want: `^192\.0\.2\.1 - bo\\x22b ` + clfTime + ` "GET /path\?q=1 HTTP/1\.1" 200 1$`, + }, + { + name: "combined adds referer and user agent", + combined: true, + handler: func(http.ResponseWriter, *http.Request) {}, + setup: func(r *http.Request) { + r.Header.Set("Referer", "http://ref/") + r.Header.Set("User-Agent", `ua "1"`) + }, + want: `^192\.0\.2\.1 - - ` + clfTime + ` "GET /path\?q=1 HTTP/1\.1" 200 - "http://ref/" "ua \\x221\\x22"$`, + }, + { + name: "combined with missing headers", + combined: true, + handler: func(http.ResponseWriter, *http.Request) {}, + setup: func(r *http.Request) { r.Header.Del("User-Agent") }, + want: `^192\.0\.2\.1 - - ` + clfTime + ` "GET /path\?q=1 HTTP/1\.1" 200 - "-" "-"$`, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + var buf bytes.Buffer + logger := log.New(&buf, "", 0) + req := httptest.NewRequest(http.MethodGet, "/path?q=1", nil) + req.RemoteAddr = "192.0.2.1:5555" + if tc.setup != nil { + tc.setup(req) + } + w := httptest.NewRecorder() + AccessLog(logger, tc.combined, tc.handler).ServeHTTP(w, req) + + line := strings.TrimSuffix(buf.String(), "\n") + if !regexp.MustCompile(tc.want).MatchString(line) { + t.Fatalf("log line\n %s\ndoes not match\n %s", line, tc.want) + } + }) + } +} diff --git a/internal/logging/logging.go b/internal/logging/logging.go index 098ca63..ad198fb 100644 --- a/internal/logging/logging.go +++ b/internal/logging/logging.go @@ -25,6 +25,9 @@ func EnableDebug() { } } +// LogToFile sends all logging to logfile as JSON, replacing the default slog +// logger. The file is created if needed and appended to; any log file opened +// by an earlier call is closed. func LogToFile(logfile string) error { if fileLogger != nil { (*fileLogger).Close() @@ -47,6 +50,7 @@ func LogToFile(logfile string) error { return nil } +// Shutdown closes the log file opened by LogToFile, if any. func Shutdown() error { var err error = nil if fileLogger != nil { @@ -56,6 +60,8 @@ func Shutdown() error { return err } +// LogToTerminal sends all logging to stdout as coloured text, replacing the +// default slog logger. func LogToTerminal() { opts := &tint.Options{ Level: &logLevel, @@ -72,10 +78,12 @@ func IsJSONHandler(h slog.Handler) bool { return ok } +// IsFileLogger reports whether logging currently goes to a file. func IsFileLogger() bool { return fileLogger != nil } +// IsDebugEnabled reports whether debug messages are logged. func IsDebugEnabled() bool { return logLevel.Level() <= slog.LevelDebug } diff --git a/internal/logging/logging_test.go b/internal/logging/logging_test.go new file mode 100644 index 0000000..9606f7c --- /dev/null +++ b/internal/logging/logging_test.go @@ -0,0 +1,157 @@ +package logging + +import ( + "context" + "io" + "log/slog" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/lmittmann/tint" +) + +// saveGlobals restores the package level and the default slog logger after a +// test that changes them. +func saveGlobals(t *testing.T) { + t.Helper() + level := logLevel.Level() + def := slog.Default() + t.Cleanup(func() { + logLevel.Set(level) + slog.SetDefault(def) + Shutdown() + }) +} + +func TestLevels(t *testing.T) { + saveGlobals(t) + slog.SetDefault(slog.New(slog.NewTextHandler(io.Discard, nil))) + + SetLevel(slog.LevelInfo) + if IsDebugEnabled() { + t.Fatal("debug enabled at info level") + } + EnableDebug() + if !IsDebugEnabled() { + t.Fatal("EnableDebug did not enable debug") + } + + // EnableDebug must not raise a level that is already more verbose. + SetLevel(slog.LevelDebug - 4) + EnableDebug() + if got := logLevel.Level(); got != slog.LevelDebug-4 { + t.Fatalf("level = %v, want %v", got, slog.LevelDebug-4) + } + + SetLevel(slog.LevelError) + if IsDebugEnabled() { + t.Fatal("debug enabled at error level") + } +} + +// wrapper is a handler that wraps another, as middleware handlers do. +type wrapper struct{ slog.Handler } + +func TestIsJSONHandler(t *testing.T) { + tests := []struct { + name string + h slog.Handler + want bool + }{ + {"json", slog.NewJSONHandler(io.Discard, nil), true}, + {"text", slog.NewTextHandler(io.Discard, nil), false}, + {"tint", tint.NewTextHandler(io.Discard, nil), false}, + {"nil", nil, false}, + {"json with attrs", slog.NewJSONHandler(io.Discard, nil).WithAttrs( + []slog.Attr{slog.String("k", "v")}), true}, + // Only slog's own JSON handler is recognised, not one wrapped in another. + {"wrapped json", wrapper{slog.NewJSONHandler(io.Discard, nil)}, false}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + if got := IsJSONHandler(tc.h); got != tc.want { + t.Fatalf("IsJSONHandler = %v, want %v", got, tc.want) + } + }) + } +} + +func TestLogToFileWritesJSON(t *testing.T) { + saveGlobals(t) + SetLevel(slog.LevelInfo) + path := filepath.Join(t.TempDir(), "app.log") + + if err := LogToFile(path); err != nil { + t.Fatal(err) + } + if st, err := os.Stat(path); err != nil { + t.Fatal(err) + } else if perm := st.Mode().Perm(); perm != 0o600 { + t.Errorf("log file permissions = %o, want 600", perm) + } + if !IsJSONHandler(slog.Default().Handler()) { + t.Fatal("default handler is not JSON after LogToFile") + } + slog.Info("hello file", "k", "v") + slog.Debug("hidden") + + //nolint:gosec // G304: reading a file the test created in t.TempDir + data, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + s := string(data) + if !strings.Contains(s, `"msg":"hello file"`) || !strings.Contains(s, `"k":"v"`) { + t.Fatalf("log file missing entry: %s", s) + } + if strings.Contains(s, "hidden") { + t.Fatalf("debug line written at info level: %s", s) + } +} + +func TestLogToFileBadPath(t *testing.T) { + saveGlobals(t) + if err := LogToFile(filepath.Join(t.TempDir(), "missing", "app.log")); err == nil { + t.Fatal("expected an error for a path in a missing directory") + } +} + +func TestLogToFileTracksFile(t *testing.T) { + saveGlobals(t) + if err := LogToFile(filepath.Join(t.TempDir(), "app.log")); err != nil { + t.Fatal(err) + } + if !IsFileLogger() { + t.Fatal("IsFileLogger() = false after LogToFile") + } + if err := Shutdown(); err != nil { + t.Fatal(err) + } + if IsFileLogger() { + t.Fatal("IsFileLogger() = true after Shutdown") + } +} + +func TestShutdownWithoutFile(t *testing.T) { + if err := Shutdown(); err != nil { + t.Fatal(err) + } + if IsFileLogger() { + t.Fatal("IsFileLogger() = true with no file") + } +} + +func TestLogToTerminal(t *testing.T) { + saveGlobals(t) + LogToTerminal() + h := slog.Default().Handler() + if IsJSONHandler(h) { + t.Fatal("terminal handler reported as JSON") + } + SetLevel(slog.LevelWarn) + if h.Enabled(context.Background(), slog.LevelInfo) { + t.Fatal("terminal handler ignores the package level") + } +} diff --git a/internal/mock/README.md b/internal/mock/README.md new file mode 100644 index 0000000..034c4b4 --- /dev/null +++ b/internal/mock/README.md @@ -0,0 +1,115 @@ +# mock + +Mock versions of the external services JobSearch fetches from, for local +development and testing without hitting the real APIs or their rate limits. + +| Package | Mocks | Command | +|---|---|---| +| `findATender` | UK Government Find a Tender service (FTS) OCDS API | `cmd/mock-fts` | + + +## findATender +A mock of the FTS release package endpoint: + +``` +GET /api/1.0/ocdsReleasePackages?updatedFrom=2026-10-06T00:00:00&limit=5 +``` + +It generates realistic, fictional OCDS releases from a catalog of buyers, +suppliers and tender subjects, keeps publishing new ones while it runs, and +behaves like the real service: paging, query validation and rate limiting. +The JobSearch `fts_mock` source points at it. + + +### Running +```bash +go run ./cmd/mock-fts --addr :8080 +``` + +Then enable the `fts_mock` source in `config.yaml`: +```yaml +sources: + fts_mock: + enabled: true + interval: 1m + api_root: http://localhost:8080/api/1.0 +``` + +Useful flags (`go run ./cmd/mock-fts --help` lists them all): + +| Flag | Default | Meaning | +|---|---|---| +| `--addr` | `:8080` | Listen address | +| `--base-url` | `http://localhost:8080` | Public base URL used in `uri` and `links.next` | +| `--data-dir` | `./mock-data` | Where `releases.jsonl` is stored | +| `--reset` | | Delete stored releases and regenerate the backfill | +| `--backfill` | `72h` | History to generate on an empty data directory | +| `--min-gap`, `--max-gap` | `30s`, `5m` | Range of time between generated releases | +| `--rate-limit`, `--rate-window` | `12`, `5m` | Requests allowed per client per window | +| `--access-log` | `-` (stdout) | Access log file | +| `--log-format` | `combined` | Access log format: `combined` or `common` | +| `--tz` | `Europe/London` | Time zone for query times with no offset | + +For quick testing, shorten the gaps and raise the rate limit: +```bash +go run ./cmd/mock-fts --min-gap 5s --max-gap 10s --rate-limit 1000 +``` + + +### API behaviour +Query parameters, as on the real service: + +| Parameter | Format | Meaning | +|---|---|---| +| `updatedFrom` | `yyyy-MM-ddTHH:mm:ss` | Releases dated at or after this time | +| `updatedTo` | `yyyy-MM-ddTHH:mm:ss` | Releases dated at or before this time (optional) | +| `limit` | `1`–`100`, default `100` | Page size | +| `cursor` | opaque | Next page; taken from `links.next` | + +- Like FTS, query times are UK local time (BST in summer) unless they carry an + explicit offset or `Z`. Release dates are stored and returned in UTC. +- Results are in date order. When there are more, `links.next` holds the URL + of the next page, with the original query and a `cursor`. +- Releases dated after the current time are never returned. +- Invalid parameters get `400 Bad Request` with a message, and methods other + than `GET` get `405 Method Not Allowed`. +- Each client (by remote IP) may make `--rate-limit` requests per + `--rate-window`. Windows are fixed and start at the client's first request + in the current clock hour. Over the limit the mock returns + `429 Too Many Requests` with a `Retry-After` header and the body + `Rate limit of N exceeded. Please retry after X seconds.` + + +### Generated data +- **Catalog** (`findATender/data/catalog.json`): the buyers, suppliers, + tender subjects, procurement methods and classifications that releases are + drawn from. It's embedded in the binary, so rebuild after changing it. + Subjects refer to classifications by `{scheme}-{id}` keys (e.g. + `CPV-90513000`). The mock refuses to start if the catalog has an empty pool + or a key that doesn't resolve. +- **Procedures**: a new release usually starts a new procedure, with a new + `ocds-mock-` ocid, as a `tender` (or sometimes `planning`). About a third of + the time it moves an open procedure on instead, from `planning` to `tender`, + `tender` to `tenderUpdate` (a deadline extension) or `award`. So the same + ocid appears in several releases, as it does on the real service. +- **Store** (`/releases.jsonl`): every release is appended as one + line of JSON, so releases survive restarts. On startup the mock reloads the + file, skips any corrupt lines, and generates releases to fill the time since + the last one (or `--backfill`, if that's shorter). + + +### Code + +| File | Contents | +|---|---| +| `doc.go` | Package documentation and data file names | +| `catalog.go` | Loading and validating the embedded catalog | +| `generator.go` | Making releases, catching up on startup, and the background publisher | +| `store.go` | The in-memory, file-backed release store | +| `handler.go` | The HTTP handler: query parsing, paging and cursors | +| `ratelimit.go` | The per-client rate limiter and its middleware | +| `ocds/` | Generated OCDS types (`ocds_gen.go`, from `ocdsgen`); don't edit by hand | +| `testdata/` | A small catalog used by the tests | + +The mock's `ocds` package is separate from `internal/findATender/ocds`, which +the JobSearch client uses. diff --git a/internal/mock/findATender/catalog.go b/internal/mock/findATender/catalog.go index 1b448da..31b5db3 100644 --- a/internal/mock/findATender/catalog.go +++ b/internal/mock/findATender/catalog.go @@ -73,6 +73,9 @@ func parseCatalog(b []byte, name string) (*Catalog, error) { return &c, nil } +// validate checks that every pool has at least one entry and that every +// classification key used by a subject or its items is in Classifications. +// It reports all the problems found, listing at most five missing keys. func (c *Catalog) validate() error { var errs []error @@ -156,6 +159,8 @@ func (c *Catalog) subject(r *rand.Rand) ocds.Tender { return t } +// resolveOne returns a copy of the classification with key, or nil if there +// is none. func (c *Catalog) resolveOne(key string) *ocds.Classification { if cl, ok := c.Classifications[key]; ok { return &cl diff --git a/internal/mock/findATender/catalog_test.go b/internal/mock/findATender/catalog_test.go new file mode 100644 index 0000000..0c72b2d --- /dev/null +++ b/internal/mock/findATender/catalog_test.go @@ -0,0 +1,134 @@ +package mock + +import ( + "math/rand" + "os" + "strings" + "testing" +) + +// loadTestCatalog parses the small fixture catalog used by the tests. +func loadTestCatalog(t *testing.T) *Catalog { + t.Helper() + b, err := os.ReadFile("testdata/catalog.json") + if err != nil { + t.Fatal(err) + } + cat, err := parseCatalog(b, "testdata/catalog.json") + if err != nil { + t.Fatal(err) + } + return cat +} + +// The catalog compiled into the build must load: a bad data/catalog.json +// fails here rather than when the mock starts. +func TestEmbeddedCatalogIsValid(t *testing.T) { + if len(embeddedCatalog) == 0 { + t.Fatal("data/catalog.json was not embedded") + } + cat, err := LoadCatalog() + if err != nil { + t.Fatal(err) + } + if len(cat.Buyers) == 0 || len(cat.Suppliers) == 0 || len(cat.Subjects) == 0 || + len(cat.Methods) == 0 { + t.Fatalf("embedded catalog has an empty pool: %d buyers, %d suppliers, %d subjects, %d methods", + len(cat.Buyers), len(cat.Suppliers), len(cat.Subjects), len(cat.Methods)) + } +} + +func TestParseCatalog(t *testing.T) { + cat := loadTestCatalog(t) + if len(cat.Buyers) != 2 || len(cat.Suppliers) != 1 || len(cat.Subjects) != 2 || + len(cat.Methods) != 2 { + t.Fatalf("got %d buyers, %d suppliers, %d subjects, %d methods", + len(cat.Buyers), len(cat.Suppliers), len(cat.Subjects), len(cat.Methods)) + } + if d := cat.Buyers[0].Details; d == nil || len(d.Classifications) != 1 || + d.Classifications[0].Scheme != "UK_CA_TYPE" { + t.Fatalf("buyer details.classifications not read: %+v", d) + } + s := cat.Subjects[0] + if s.Classification != "CPV-77314000" || len(s.Items) != 1 { + t.Fatalf("subject classification key/items not read: %+v", s) + } + if ac := s.Items[0].AdditionalClassifications; len(ac) != 1 || ac[0] != "CPV-77310000" { + t.Fatalf("items.additionalClassifications keys not read: %+v", ac) + } + if v := s.Lots[0].Value; v == nil || v.AmountGross == nil || *v.AmountGross != 120000 { + t.Fatalf("lot value not read: %+v", v) + } +} + +// Picking a subject resolves its classification keys to full classifications. +func TestSubjectResolvesClassifications(t *testing.T) { + cat := loadTestCatalog(t) + //nolint:gosec // G404: seeded math/rand keeps the test deterministic + r := rand.New(rand.NewSource(1)) + for i := 0; i < 20; i++ { + tender := cat.subject(r) + if tender.Title != "Grounds maintenance services" { + continue + } + if cl := tender.Classification; cl == nil || cl.ID != "77314000" || + cl.Description != "Grounds maintenance services" { + t.Fatalf("main classification = %+v", cl) + } + ac := tender.Items[0].AdditionalClassifications + if len(ac) != 1 || ac[0].Description != "Planting and maintenance services of green areas" { + t.Fatalf("item classifications = %+v", ac) + } + // The catalog itself still holds keys, untouched by picking. + if cat.Subjects[0].Classification != "CPV-77314000" { + t.Fatal("picking modified the catalog") + } + return + } + t.Fatal("never picked the grounds maintenance subject") +} + +func TestParseCatalogRejectsInvalidCatalogs(t *testing.T) { + cases := map[string]struct { + doc string + want []string + }{ + "empty pools": { + doc: `{"buyers": [], "subjects": [{"title": "T"}], "methods": [], "classifications": {}}`, + want: []string{"no buyers", "no suppliers", "no methods"}, + }, + "unknown classification keys": { + doc: `{"buyers": [{"name": "B"}], "suppliers": [{"name": "S"}], "methods": [{"procurementMethod": "open"}], + "subjects": [{"title": "T", "classification": "CPV-1", "items": [{"id": "1", "additionalClassifications": ["CPV-2"]}]}], + "classifications": {}}`, + want: []string{"2 classification key(s) not found", "CPV-1", "CPV-2"}, + }, + "not JSON": {doc: `{`, want: []string{"decoding"}}, + } + for name, tc := range cases { + t.Run(name, func(t *testing.T) { + _, err := parseCatalog([]byte(tc.doc), "test catalog") + if err == nil { + t.Fatal("expected an error") + } + for _, w := range tc.want { + if !strings.Contains(err.Error(), w) { + t.Errorf("error %q does not mention %q", err, w) + } + } + }) + } +} + +func TestResolveOneUnknownKey(t *testing.T) { + cat := loadTestCatalog(t) + if cl := cat.resolveOne("CPV-00000000"); cl != nil { + t.Errorf("unknown key resolved to %+v", cl) + } + if cl := cat.resolveOne(""); cl != nil { + t.Errorf("empty key resolved to %+v", cl) + } + if got := cat.resolve([]string{"CPV-00000000"}); got != nil { + t.Errorf("resolve of unknown keys = %+v", got) + } +} diff --git a/internal/mock/findATender/generator.go b/internal/mock/findATender/generator.go index 5729a14..be90ed7 100644 --- a/internal/mock/findATender/generator.go +++ b/internal/mock/findATender/generator.go @@ -23,6 +23,8 @@ type Generator struct { rnd *rand.Rand } +// NewGenerator returns a generator drawing from cat and storing releases in +// store. The same seed and store contents give the same releases. func NewGenerator(cat *Catalog, store *Store, seed int64) *Generator { return &Generator{ cat: cat, @@ -31,6 +33,9 @@ func NewGenerator(cat *Catalog, store *Store, seed int64) *Generator { } } +// Make returns a new release dated at (to the second). About a third of the +// time it moves on a procedure still open from the last 60 days; otherwise it +// starts a new one. func (g *Generator) Make(at time.Time) ocds.Release { at = at.UTC().Truncate(time.Second) if g.rnd.Float64() < 0.35 { @@ -105,6 +110,9 @@ func (g *Generator) newProcedure(at time.Time) ocds.Release { } } +// advance returns the next release in prev's procedure: planning moves to +// tender, and a tender either has its deadline extended (tenderUpdate) or is +// awarded to a supplier from the catalog. func (g *Generator) advance(prev ocds.Release, at time.Time) ocds.Release { r := clone(prev) r.ID = g.store.NextNoticeID(at.Year()) @@ -190,6 +198,8 @@ func (g *Generator) newOCID() string { } } +// gap returns a random duration from min to max inclusive, or min if max is +// not larger. func (g *Generator) gap(min, max time.Duration) time.Duration { if max <= min { return min diff --git a/internal/mock/findATender/generator_test.go b/internal/mock/findATender/generator_test.go new file mode 100644 index 0000000..6037332 --- /dev/null +++ b/internal/mock/findATender/generator_test.go @@ -0,0 +1,215 @@ +package mock + +import ( + "context" + "io" + "log" + "math/rand" + "path/filepath" + "strings" + "testing" + "time" + + "git.avsdev.uk/Go/job-search/internal/mock/findATender/ocds" +) + +func quietLog(t *testing.T) { + t.Helper() + w := log.Writer() + log.SetOutput(io.Discard) + t.Cleanup(func() { log.SetOutput(w) }) +} + +func TestGap(t *testing.T) { + //nolint:gosec // G404: seeded math/rand keeps the test deterministic + g := &Generator{rnd: rand.New(rand.NewSource(1))} + if d := g.gap(time.Second, time.Second); d != time.Second { + t.Errorf("min == max: %s", d) + } + if d := g.gap(2*time.Second, time.Second); d != 2*time.Second { + t.Errorf("max < min: %s, want min", d) + } + for i := 0; i < 1000; i++ { + if d := g.gap(time.Second, 2*time.Second); d < time.Second || d > 2*time.Second { + t.Fatalf("gap out of range: %s", d) + } + } +} + +func TestScaleValue(t *testing.T) { + scaleValue(nil, 2) // must not panic + + v := &ocds.Value{Amount: ptr(12345.0), Currency: "GBP"} + scaleValue(v, 1.5) + if *v.Amount != 18500 { // 18517.5 rounded to the nearest 100 + t.Errorf("amount = %v, want 18500", *v.Amount) + } + if v.AmountGross != nil { + t.Error("nil gross amount was set") + } + + v = &ocds.Value{Amount: ptr(1000.0), AmountGross: ptr(1200.0)} + scaleValue(v, 0.5) + if *v.Amount != 500 || *v.AmountGross != 600 { + t.Errorf("amounts = %v / %v, want 500 / 600", *v.Amount, *v.AmountGross) + } +} + +func TestEnsurePartyID(t *testing.T) { + p := ocds.Organization{ID: "GB-EXISTING", Name: "Council"} + ensurePartyID(&p) + if p.ID != "GB-EXISTING" { + t.Errorf("existing id replaced: %s", p.ID) + } + + a := ocds.Organization{Name: "Somewhere Council"} + b := ocds.Organization{Name: "Somewhere Council"} + c := ocds.Organization{Name: "Elsewhere Council"} + ensurePartyID(&a) + ensurePartyID(&b) + ensurePartyID(&c) + if !strings.HasPrefix(a.ID, "GB-FTS-") { + t.Errorf("id = %q", a.ID) + } + if a.ID != b.ID { + t.Errorf("same name gave %s and %s", a.ID, b.ID) + } + if a.ID == c.ID { + t.Errorf("different names gave the same id %s", a.ID) + } +} + +func TestAdvance(t *testing.T) { + cat := loadTestCatalog(t) + store, err := OpenStore(filepath.Join(t.TempDir(), "r.jsonl")) + if err != nil { + t.Fatal(err) + } + defer store.Close() + g := NewGenerator(cat, store, 1) + at := time.Date(2026, 10, 8, 12, 0, 0, 0, time.UTC) + + // A planning notice without a tender becomes an active tender. + r := g.advance(ocds.Release{OCID: "ocds-x", Tag: []string{"planning"}}, at) + if r.Tag[0] != "tender" || r.Tender == nil || r.Tender.Status != "active" || + r.Tender.TenderPeriod == nil || r.Tender.TenderPeriod.EndDate == nil { + t.Fatalf("planning -> %v, tender %+v", r.Tag, r.Tender) + } + if string(r.Tender.ID) != "ocds-x" || !r.Date.Equal(at) { + t.Errorf("tender id %s, date %s", r.Tender.ID, r.Date) + } + + // A tender is either updated (deadline + 14 days) or awarded. + end := at.AddDate(0, 0, 20) + prev := ocds.Release{ + OCID: "ocds-y", + Tag: []string{"tender"}, + Tender: &ocds.Tender{ + ID: "ocds-y", + Value: &ocds.Value{Amount: ptr(10000.0), Currency: "GBP"}, + Lots: []ocds.Lot{{ID: "1"}, {ID: ""}}, + TenderPeriod: &ocds.Period{EndDate: &end}, + }, + } + seen := map[string]bool{} + for i := 0; i < 50; i++ { + r := g.advance(prev, at) + seen[r.Tag[0]] = true + switch r.Tag[0] { + case "tenderUpdate": + if got := *r.Tender.TenderPeriod.EndDate; !got.Equal(end.AddDate(0, 0, 14)) { + t.Fatalf("updated deadline %s", got) + } + case "award": + if r.Tender.Status != "complete" || len(r.Awards) != 1 { + t.Fatalf("award: status %s, %d awards", r.Tender.Status, len(r.Awards)) + } + aw := r.Awards[0] + if aw.Value == nil || *aw.Value.Amount < 8000 || *aw.Value.Amount > 10000 { + t.Fatalf("award value %+v", aw.Value) + } + if len(aw.RelatedLots) != 1 || aw.RelatedLots[0] != "1" { + t.Fatalf("related lots %v", aw.RelatedLots) + } + if len(aw.Suppliers) != 1 || len(r.Parties) != 1 || r.Parties[0].Roles[0] != "supplier" { + t.Fatalf("suppliers %v, parties %v", aw.Suppliers, r.Parties) + } + default: + t.Fatalf("tender -> %s", r.Tag[0]) + } + } + if !seen["tenderUpdate"] || !seen["award"] { + t.Errorf("expected both updates and awards, saw %v", seen) + } + if *prev.Tender.TenderPeriod.EndDate != end || prev.Awards != nil { + t.Error("advance modified the previous release") + } + + // An update without a deadline gets 30 + 14 days; a tender with no value + // is awarded with no value. + prev = ocds.Release{OCID: "ocds-z", Tag: []string{"tender"}, Tender: &ocds.Tender{ID: "ocds-z"}} + for i := 0; i < 50; i++ { + r := g.advance(prev, at) + switch r.Tag[0] { + case "tenderUpdate": + if got := *r.Tender.TenderPeriod.EndDate; !got.Equal(at.AddDate(0, 0, 44)) { + t.Fatalf("default deadline %s", got) + } + case "award": + if r.Awards[0].Value != nil { + t.Fatalf("award value %+v, want nil", r.Awards[0].Value) + } + } + } +} + +func TestRunPublishesUntilCancelled(t *testing.T) { + quietLog(t) + cat := loadTestCatalog(t) + store, err := OpenStore(filepath.Join(t.TempDir(), "r.jsonl")) + if err != nil { + t.Fatal(err) + } + defer store.Close() + + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan struct{}) + go func() { + NewGenerator(cat, store, 1).Run(ctx, time.Millisecond, 2*time.Millisecond) + close(done) + }() + + deadline := time.After(5 * time.Second) + for store.Len() < 3 { + select { + case <-deadline: + cancel() + t.Fatalf("only %d releases published", store.Len()) + case <-time.After(time.Millisecond): + } + } + cancel() + select { + case <-done: + case <-time.After(5 * time.Second): + t.Fatal("Run did not return after cancel") + } +} + +// A failing store is logged and Run carries on until cancelled. +func TestRunSurvivesStoreErrors(t *testing.T) { + quietLog(t) + cat := loadTestCatalog(t) + store, err := OpenStore(filepath.Join(t.TempDir(), "r.jsonl")) + if err != nil { + t.Fatal(err) + } + store.Close() + + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond) + defer cancel() + NewGenerator(cat, store, 1).Run(ctx, time.Millisecond, time.Millisecond) + if store.Len() != 0 { + t.Errorf("Len = %d, want 0", store.Len()) + } +} diff --git a/internal/mock/findATender/handler.go b/internal/mock/findATender/handler.go index 133cb9d..8a27bf2 100644 --- a/internal/mock/findATender/handler.go +++ b/internal/mock/findATender/handler.go @@ -43,6 +43,8 @@ func encodeCursor(i int) string { return base64.RawURLEncoding.EncodeToString([]byte("idx:" + strconv.Itoa(i))) } +// decodeCursor returns the release index in a cursor made by encodeCursor. +// Anything else, including a negative index, is an error. func decodeCursor(c string) (int, error) { b, err := base64.RawURLEncoding.DecodeString(c) if err != nil || !strings.HasPrefix(string(b), "idx:") { diff --git a/internal/mock/findATender/handler_test.go b/internal/mock/findATender/handler_test.go new file mode 100644 index 0000000..2a8a68e --- /dev/null +++ b/internal/mock/findATender/handler_test.go @@ -0,0 +1,209 @@ +package mock + +import ( + "encoding/base64" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "path/filepath" + "strings" + "testing" + "time" + + "git.avsdev.uk/Go/job-search/internal/mock/findATender/ocds" +) + +func TestUpdatedFromIsLocalTime(t *testing.T) { + london, err := time.LoadLocation("Europe/London") + if err != nil { + t.Fatal(err) + } + store, err := OpenStore(filepath.Join(t.TempDir(), "r.jsonl")) + if err != nil { + t.Fatal(err) + } + defer store.Close() + + // Two releases at 11:45Z and 12:15Z, i.e. 12:45 and 13:15 BST. + for i, at := range []time.Time{ + time.Date(2026, 10, 8, 11, 45, 0, 0, time.UTC), + time.Date(2026, 10, 8, 12, 15, 0, 0, time.UTC), + } { + if err := store.Append( + ocds.Release{ + OCID: "ocds-test", + ID: []string{"000001-2026", "000002-2026"}[i], + Date: at, + Tag: []string{"tender"}, + }, + ); err != nil { + t.Fatal(err) + } + } + + h := NewHandler(store, "", london) + count := func(query string) int { + rec := httptest.NewRecorder() + h(rec, httptest.NewRequest("GET", BasePath+"?"+query, nil)) + var pkg ocds.ReleasePackage + if err := json.Unmarshal(rec.Body.Bytes(), &pkg); err != nil { + t.Fatalf("%s: %v (%s)", query, err, rec.Body.String()) + } + return len(pkg.Releases) + } + + cases := map[string]int{ + "updatedFrom=2026-10-08T13:00:00": 1, // 13:00 BST = 12:00Z + "updatedFrom=2026-10-08T12:30:00": 2, // 12:30 BST = 11:30Z + "updatedFrom=2026-10-08T12:00:00Z": 1, // explicit UTC still honoured + "updatedFrom=2026-10-08T12:00:00%2B01:00": 2, // explicit offset still honoured + "updatedFrom=2026-10-08T12:00:00&updatedTo=2026-10-08T13:00:00": 1, // window in BST + } + for q, want := range cases { + if got := count(q); got != want { + t.Errorf("%s: got %d releases, want %d", q, got, want) + } + } + + // In winter (GMT) London local time equals UTC. + winter := time.Date(2026, 12, 1, 9, 0, 0, 0, time.UTC) + if got, _ := parseTime("2026-12-01T09:00:00", london); !got.Equal(winter) { + t.Errorf("GMT: got %s, want %s", got, winter) + } +} + +func openTestStore(t *testing.T) *Store { + t.Helper() + store, err := OpenStore(filepath.Join(t.TempDir(), "r.jsonl")) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { store.Close() }) + return store +} + +func TestHandlerRejectsBadRequests(t *testing.T) { + h := NewHandler(openTestStore(t), "", nil) + + rec := httptest.NewRecorder() + h(rec, httptest.NewRequest("POST", BasePath, nil)) + if rec.Code != http.StatusMethodNotAllowed { + t.Errorf("POST: status %d, want 405", rec.Code) + } + + cursor := func(s string) string { return base64.RawURLEncoding.EncodeToString([]byte(s)) } + cases := map[string]string{ + "limit=0": "limit must be an integer between 1 and 100", + "limit=101": "limit must be an integer between 1 and 100", + "limit=ten": "limit must be an integer between 1 and 100", + "updatedFrom=yesterday": "updatedFrom must be in format yyyy-MM-ddTHH:mm:ss", + "updatedTo=2026-13-01": "updatedTo must be in format yyyy-MM-ddTHH:mm:ss", + "updatedFrom=2026-10-08T12:00:00&updatedTo=2026-10-08T11:00:00": "updatedTo must not be before updatedFrom", + "cursor=!!!": "invalid cursor", + "cursor=" + cursor("pos:1"): "invalid cursor", + "cursor=" + cursor("idx:one"): "invalid cursor", + "cursor=" + cursor("idx:-1"): "invalid cursor", + } + for q, want := range cases { + rec := httptest.NewRecorder() + h(rec, httptest.NewRequest("GET", BasePath+"?"+q, nil)) + if rec.Code != http.StatusBadRequest { + t.Errorf("%s: status %d, want 400", q, rec.Code) + continue + } + if ct := rec.Header().Get("Content-Type"); ct != "application/json" { + t.Errorf("%s: Content-Type %q", q, ct) + } + var body map[string]string + if err := json.Unmarshal(rec.Body.Bytes(), &body); err != nil || body["error"] != want { + t.Errorf("%s: body %s, want error %q", q, rec.Body.String(), want) + } + } +} + +func TestCursorRoundTrip(t *testing.T) { + for _, i := range []int{0, 1, 99, 123456} { + if got, err := decodeCursor(encodeCursor(i)); err != nil || got != i { + t.Errorf("decodeCursor(encodeCursor(%d)) = %d, %v", i, got, err) + } + } +} + +func TestParseTimeFormats(t *testing.T) { + cases := map[string]time.Time{ + "2026-10-08T12:00:00Z": time.Date(2026, 10, 8, 12, 0, 0, 0, time.UTC), + "2026-10-08T12:00:00+02:00": time.Date(2026, 10, 8, 10, 0, 0, 0, time.UTC), + "2026-10-08T12:00:00": time.Date(2026, 10, 8, 12, 0, 0, 0, time.UTC), + "2026-10-08": time.Date(2026, 10, 8, 0, 0, 0, 0, time.UTC), + } + for v, want := range cases { + got, err := parseTime(v, nil) // nil location means UTC + if err != nil || !got.Equal(want) { + t.Errorf("parseTime(%q) = %s, %v; want %s", v, got, err, want) + } + } + for _, v := range []string{"", "08/10/2026", "2026-10-08 12:00:00", "2026-10-08T25:00:00"} { + if _, err := parseTime(v, nil); err == nil { + t.Errorf("parseTime(%q): expected error", v) + } + } +} + +// Following links.next walks every release exactly once, keeping the other +// query parameters, and the last page has no next link. +func TestHandlerPagination(t *testing.T) { + store := openTestStore(t) + base := time.Date(2026, 10, 8, 9, 0, 0, 0, time.UTC) + for i := 0; i < 5; i++ { + r := ocds.Release{ + OCID: fmt.Sprintf("ocds-test-%d", i), + ID: fmt.Sprintf("%06d-2026", i+1), + Date: base.Add(time.Duration(i) * time.Minute), + Tag: []string{"tender"}, + } + if err := store.Append(r); err != nil { + t.Fatal(err) + } + } + + const baseURL = "http://mock.test" + h := NewHandler(store, baseURL, nil) + next := BasePath + "?limit=2&updatedFrom=2026-10-08T09:01:00" + var ids []string + for pages := 0; next != ""; pages++ { + if pages > 5 { + t.Fatal("too many pages") + } + rec := httptest.NewRecorder() + h(rec, httptest.NewRequest("GET", next, nil)) + if rec.Code != http.StatusOK { + t.Fatalf("%s: status %d: %s", next, rec.Code, rec.Body.String()) + } + var pkg ocds.ReleasePackage + if err := json.Unmarshal(rec.Body.Bytes(), &pkg); err != nil { + t.Fatal(err) + } + if pkg.URI != baseURL+next { + t.Errorf("uri = %q, want %q", pkg.URI, baseURL+next) + } + for _, r := range pkg.Releases { + ids = append(ids, r.ID) + } + next = "" + if pkg.Links != nil && pkg.Links.Next != "" { + if !strings.HasPrefix(pkg.Links.Next, baseURL+BasePath+"?") { + t.Fatalf("next = %q", pkg.Links.Next) + } + if !strings.Contains(pkg.Links.Next, "limit=2") || + !strings.Contains(pkg.Links.Next, "updatedFrom=") { + t.Errorf("next lost query parameters: %q", pkg.Links.Next) + } + next = strings.TrimPrefix(pkg.Links.Next, baseURL) + } + } + want := "000002-2026,000003-2026,000004-2026,000005-2026" + if got := strings.Join(ids, ","); got != want { + t.Errorf("walked %s, want %s", got, want) + } +} diff --git a/internal/mock/findATender/ratelimit_test.go b/internal/mock/findATender/ratelimit_test.go new file mode 100644 index 0000000..37946a5 --- /dev/null +++ b/internal/mock/findATender/ratelimit_test.go @@ -0,0 +1,134 @@ +package mock + +import ( + "net/http" + "net/http/httptest" + "strconv" + "strings" + "testing" + "time" +) + +func TestRateLimitWindows(t *testing.T) { + now := time.Date(2026, 10, 6, 10, 3, 20, 0, time.UTC) // first request of the hour + rl := NewRateLimiter(12, 5*time.Minute) + rl.now = func() time.Time { return now } + + for i := 0; i < 12; i++ { + if ok, _ := rl.allow("a"); !ok { + t.Fatalf("request %d rejected", i+1) + } + } + now = now.Add(100 * time.Second) // 10:05:00, window ends 10:08:20 + if ok, secs := rl.allow("a"); ok || secs != 200 { + t.Fatalf("want reject with 200s, got ok=%v secs=%d", ok, secs) + } + if ok, _ := rl.allow("b"); !ok { + t.Fatal("other client should be unaffected") + } + now = time.Date(2026, 10, 6, 10, 8, 20, 0, time.UTC) // next window opens + if ok, _ := rl.allow("a"); !ok { + t.Fatal("new window should allow") + } + + // Window 10:58:20-11:03:20 is cut short by the new hour at 11:00:00. + now = time.Date(2026, 10, 6, 10, 58, 30, 0, time.UTC) + for i := 0; i < 12; i++ { + rl.allow("a") + } + if ok, secs := rl.allow("a"); ok || secs != 90 { + t.Fatalf("want reject with 90s, got ok=%v secs=%d", ok, secs) + } + now = time.Date(2026, 10, 6, 11, 0, 5, 0, time.UTC) // re-anchors here + if ok, _ := rl.allow("a"); !ok { + t.Fatal("new hour should allow") + } +} + +func TestClientKey(t *testing.T) { + cases := map[string]string{ + "192.0.2.1:1234": "192.0.2.1", + "[::1]:8080": "::1", + "no-port": "no-port", // not host:port, used as-is + } + for addr, want := range cases { + req := httptest.NewRequest("GET", "/", nil) + req.RemoteAddr = addr + if got := clientKey(req); got != want { + t.Errorf("clientKey(%q) = %q, want %q", addr, got, want) + } + } +} + +func TestRateLimitMiddleware(t *testing.T) { + now := time.Date(2026, 10, 6, 10, 0, 0, 0, time.UTC) + rl := NewRateLimiter(2, 5*time.Minute) + rl.now = func() time.Time { return now } + + calls := 0 + h := rl.Middleware(func(w http.ResponseWriter, req *http.Request) { + calls++ + w.WriteHeader(http.StatusOK) + }) + do := func(addr string) *httptest.ResponseRecorder { + req := httptest.NewRequest("GET", "/", nil) + req.RemoteAddr = addr + rec := httptest.NewRecorder() + h(rec, req) + return rec + } + + for i := 0; i < 2; i++ { + if rec := do("192.0.2.1:1000"); rec.Code != http.StatusOK { + t.Fatalf("request %d: status %d", i+1, rec.Code) + } + } + // Same host, different port: still the same client. + rec := do("192.0.2.1:2000") + if rec.Code != http.StatusTooManyRequests { + t.Fatalf("status %d, want 429", rec.Code) + } + if got := rec.Header().Get("Retry-After"); got != "300" { + t.Errorf("Retry-After = %q, want 300", got) + } + if !strings.Contains(rec.Body.String(), "Rate limit of 2 exceeded") || + !strings.Contains(rec.Body.String(), "300 seconds") { + t.Errorf("body = %q", rec.Body.String()) + } + if calls != 2 { + t.Errorf("next called %d times, want 2", calls) + } + if rec := do("192.0.2.2:1000"); rec.Code != http.StatusOK { + t.Errorf("other client: status %d", rec.Code) + } +} + +// Once more than 10000 clients are tracked, a new client in a new hour drops +// every client left over from earlier hours. +func TestRateLimitPrunesStaleClients(t *testing.T) { + now := time.Date(2026, 10, 6, 10, 30, 0, 0, time.UTC) + rl := NewRateLimiter(1, time.Minute) + rl.now = func() time.Time { return now } + + for i := 0; i <= 10000; i++ { + rl.allow(strconv.Itoa(i)) + } + if len(rl.clients) != 10001 { + t.Fatalf("tracking %d clients", len(rl.clients)) + } + + // Same hour: nothing is stale, so nothing is dropped. + rl.allow("same-hour") + if len(rl.clients) != 10002 { + t.Fatalf("same hour: tracking %d clients, want 10002", len(rl.clients)) + } + + now = now.Add(time.Hour) + rl.allow("new") + if len(rl.clients) != 1 { + t.Fatalf("after prune: tracking %d clients, want 1", len(rl.clients)) + } + if _, ok := rl.clients["new"]; !ok { + t.Fatal("new client was pruned") + } +} diff --git a/internal/mock/findATender/store.go b/internal/mock/findATender/store.go index 676ab94..1b0737d 100644 --- a/internal/mock/findATender/store.go +++ b/internal/mock/findATender/store.go @@ -34,6 +34,10 @@ type Store struct { openPos map[string]int } +// OpenStore loads the releases stored at path, creating the file and its +// directory if needed, and opens it for appending. Lines that can't be read +// are skipped, releases are put back in date order if needed, and a record +// left incomplete by a crash is ended so new ones start on a fresh line. func OpenStore(path string) (*Store, error) { s := &Store{latest: map[string]int{}, lastNum: map[int]int{}, openPos: map[string]int{}} if err := os.MkdirAll(filepath.Dir(path), 0o750); err != nil { @@ -115,6 +119,9 @@ func openForAppend(path string) (*os.File, error) { return f, nil } +// index records release r, at position i, in the lookups: the ocid's latest +// release, whether its procedure is still open (an award closes it), and the +// highest notice number used each year. func (s *Store) index(i int, r ocds.Release) { s.latest[r.OCID] = i if len(r.Tag) > 0 && r.Tag[0] == "award" { @@ -132,6 +139,7 @@ func (s *Store) index(i int, r ocds.Release) { } } +// closeProcedure removes ocid from the open procedures, if it is there. func (s *Store) closeProcedure(ocid string) { pos, ok := s.openPos[ocid] if !ok { @@ -151,6 +159,7 @@ func (s *Store) Append(r ocds.Release) error { return s.append(r, true) } // Used for bulk catch-up, where a sync per record would be very slow. func (s *Store) AppendNoSync(r ocds.Release) error { return s.append(r, false) } +// Sync flushes appended releases to disk. Call it after AppendNoSync. func (s *Store) Sync() error { s.mu.Lock() defer s.mu.Unlock() @@ -204,6 +213,8 @@ func (s *Store) Page(from, upper time.Time, start, limit int) ([]ocds.Release, i return out, i, more } +// Last returns the date of the newest release, or false if the store is +// empty. func (s *Store) Last() (time.Time, bool) { s.mu.RLock() defer s.mu.RUnlock() diff --git a/internal/mock/findATender/store_test.go b/internal/mock/findATender/store_test.go new file mode 100644 index 0000000..ce173a4 --- /dev/null +++ b/internal/mock/findATender/store_test.go @@ -0,0 +1,269 @@ +package mock + +import ( + "bytes" + "encoding/json" + "math/rand" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "git.avsdev.uk/Go/job-search/internal/mock/findATender/ocds" +) + +func TestStorePersistsAndGeneratorProgresses(t *testing.T) { + cat := loadTestCatalog(t) + path := filepath.Join(t.TempDir(), "releases.jsonl") + store, err := OpenStore(path) + if err != nil { + t.Fatal(err) + } + gen := NewGenerator(cat, store, 42) + now := time.Date(2026, 10, 8, 12, 0, 0, 0, time.UTC) + n, err := gen.CatchUp(now, 48*time.Hour, 3*time.Minute, 15*time.Minute) + if err != nil || n < 48*60/15 { + t.Fatalf("catch-up generated %d (err %v)", n, err) + } + + tags := map[string]int{} + ocids := map[string]int{} + all, _, _ := store.Page(time.Time{}, now, 0, 100000) + for _, r := range all { + tags[r.Tag[0]]++ + ocids[r.OCID]++ + if r.Tender == nil || r.Tender.Classification == nil { + t.Fatalf("release %s has no tender.classification", r.ID) + } + } + if tags["award"] == 0 || tags["tender"] == 0 { + t.Fatalf("expected tenders and awards, got %v", tags) + } + if len(ocids) == len(all) { + t.Fatal("expected some procedures with more than one release") + } + store.Close() + + // Reopen: everything is still there, and catch-up only fills the new gap. + store2, err := OpenStore(path) + if err != nil { + t.Fatal(err) + } + defer store2.Close() + if store2.Len() != len(all) { + t.Fatalf("after reopen: %d releases, want %d", store2.Len(), len(all)) + } + gen2 := NewGenerator(cat, store2, 7) + m, _ := gen2.CatchUp(now.Add(time.Hour), 48*time.Hour, 3*time.Minute, 15*time.Minute) + if m < 4 || m > 20 { + t.Fatalf("one hour gap produced %d releases", m) + } + if last, _ := store2.Last(); last.After(now.Add(time.Hour)) { + t.Fatal("release dated in the future") + } + + // The HTTP handler pages through the store without gaps or repeats. + srv := &server{store: store2, baseURL: "", loc: time.UTC} + seen := map[string]bool{} + next := BasePath + "?updatedFrom=2026-10-07T00:00:00&limit=7" + for pages := 0; next != ""; pages++ { + rec := httptest.NewRecorder() + srv.handle(rec, httptest.NewRequest("GET", next, nil)) + // Strict decode: the output must contain only fields the ocds types define. + var pkg ocds.ReleasePackage + dec := json.NewDecoder(bytes.NewReader(rec.Body.Bytes())) + dec.DisallowUnknownFields() + if err := dec.Decode(&pkg); err != nil { + t.Fatalf("response doesn't match ocds types: %v", err) + } + for _, r := range pkg.Releases { + if seen[r.ID] { + t.Fatalf("release %s returned twice", r.ID) + } + seen[r.ID] = true + } + next = "" + if pkg.Links != nil { + next = pkg.Links.Next + } + } + from := time.Date(2026, 10, 7, 0, 0, 0, 0, time.UTC) + want, _, _ := store2.Page(from, time.Now(), 0, 100000) + if len(seen) != len(want) { + t.Fatalf("paged %d releases, want %d", len(seen), len(want)) + } +} + +func writeLines(t *testing.T, path string, lines ...string) { + t.Helper() + if err := os.WriteFile(path, []byte(strings.Join(lines, "\n")), 0o600); err != nil { + t.Fatal(err) + } +} + +func releaseLine(t *testing.T, ocid, id, tag string, at time.Time) string { + t.Helper() + b, err := json.Marshal(ocds.Release{OCID: ocid, ID: id, Date: at, Tag: []string{tag}}) + if err != nil { + t.Fatal(err) + } + return string(b) +} + +// Unreadable lines are skipped, blank lines ignored, a hand-edited file is put +// back in date order, and the index (open procedures, notice numbers) is rebuilt. +func TestOpenStoreRecoversFromMessyFile(t *testing.T) { + path := filepath.Join(t.TempDir(), "r.jsonl") + t0 := time.Date(2026, 10, 8, 9, 0, 0, 0, time.UTC) + writeLines(t, path, + releaseLine(t, "ocds-b", "000007-2026", "tender", t0.Add(2*time.Hour)), + "{not json", + "", + releaseLine(t, "ocds-a", "000003-2026", "tender", t0), + releaseLine(t, "ocds-a", "000005-2026", "award", t0.Add(time.Hour)), + "", + ) + + store, err := OpenStore(path) + if err != nil { + t.Fatal(err) + } + defer store.Close() + + all, _, _ := store.Page(time.Time{}, t0.Add(24*time.Hour), 0, 100) + var ids []string + for _, r := range all { + ids = append(ids, r.ID) + } + if got, want := strings.Join(ids, ","), "000003-2026,000005-2026,000007-2026"; got != want { + t.Fatalf("releases %s, want %s", got, want) + } + if got := store.NextNoticeID(2026); got != "000008-2026" { + t.Errorf("NextNoticeID = %s, want 000008-2026", got) + } + if got := store.NextNoticeID(2027); got != "000001-2027" { + t.Errorf("NextNoticeID(2027) = %s", got) + } + // ocds-a was awarded, so only ocds-b is still open. + //nolint:gosec // G404: seeded math/rand keeps the test deterministic + r, ok := store.RandomOpen(rand.New(rand.NewSource(1)), t0) + if !ok || r.OCID != "ocds-b" { + t.Errorf("RandomOpen = %s, %v; want ocds-b", r.OCID, ok) + } +} + +// A record cut off mid-write is skipped, and the next record starts on a fresh +// line so it is still readable after a restart. +func TestOpenStoreAfterTruncatedWrite(t *testing.T) { + path := filepath.Join(t.TempDir(), "r.jsonl") + t0 := time.Date(2026, 10, 8, 9, 0, 0, 0, time.UTC) + full := releaseLine(t, "ocds-a", "000001-2026", "tender", t0) + cut := releaseLine(t, "ocds-b", "000002-2026", "tender", t0.Add(time.Minute)) + writeLines(t, path, full, cut[:len(cut)/2]) // no trailing newline + + store, err := OpenStore(path) + if err != nil { + t.Fatal(err) + } + if store.Len() != 1 { + t.Fatalf("Len = %d, want 1", store.Len()) + } + if err := store.Append( + ocds.Release{ + OCID: "ocds-c", + ID: "000002-2026", + Date: t0.Add(2 * time.Minute), + Tag: []string{"tender"}, + }, + ); err != nil { + t.Fatal(err) + } + store.Close() + + store2, err := OpenStore(path) + if err != nil { + t.Fatal(err) + } + defer store2.Close() + if store2.Len() != 2 || !store2.HasOCID("ocds-c") { + t.Fatalf("after reopen: Len = %d, has ocds-c = %v", store2.Len(), store2.HasOCID("ocds-c")) + } +} + +func TestOpenStoreErrors(t *testing.T) { + dir := t.TempDir() + + // The parent "directory" is a regular file. + file := filepath.Join(dir, "file") + if err := os.WriteFile(file, nil, 0o600); err != nil { + t.Fatal(err) + } + if _, err := OpenStore(filepath.Join(file, "r.jsonl")); err == nil { + t.Error("parent is a file: expected error") + } + + // The store path is a directory, which opens but can't be read. + if _, err := OpenStore(dir); err == nil { + t.Error("path is a directory: expected error") + } + + // A missing parent directory is created. + store, err := OpenStore(filepath.Join(dir, "a", "b", "r.jsonl")) + if err != nil { + t.Fatalf("nested path: %v", err) + } + store.Close() +} + +// A release dated before the newest one is moved up to keep the order valid. +func TestAppendKeepsDateOrder(t *testing.T) { + store, err := OpenStore(filepath.Join(t.TempDir(), "r.jsonl")) + if err != nil { + t.Fatal(err) + } + defer store.Close() + t0 := time.Date(2026, 10, 8, 9, 0, 0, 0, time.UTC) + store.Append(ocds.Release{OCID: "a", ID: "000001-2026", Date: t0, Tag: []string{"tender"}}) + store.Append( + ocds.Release{OCID: "b", ID: "000002-2026", Date: t0.Add(-time.Hour), Tag: []string{"tender"}}, + ) + if last, _ := store.Last(); !last.Equal(t0) { + t.Errorf("Last = %s, want %s", last, t0) + } + + // Appending to a closed store fails and leaves the store unchanged. + store.Close() + if err := store.Append(ocds.Release{OCID: "c", Date: t0, Tag: []string{"tender"}}); err == nil { + t.Error("append after close: expected error") + } + if store.Len() != 2 || store.HasOCID("c") { + t.Errorf("failed append changed the store: Len = %d", store.Len()) + } +} + +// Procedures idle since before the cut-off are dropped from the open set. +func TestRandomOpenDropsStaleProcedures(t *testing.T) { + store, err := OpenStore(filepath.Join(t.TempDir(), "r.jsonl")) + if err != nil { + t.Fatal(err) + } + defer store.Close() + t0 := time.Date(2026, 10, 8, 9, 0, 0, 0, time.UTC) + store.Append(ocds.Release{OCID: "old", ID: "000001-2026", Date: t0, Tag: []string{"tender"}}) + store.Append( + ocds.Release{OCID: "new", ID: "000002-2026", Date: t0.Add(time.Hour), Tag: []string{"tender"}}, + ) + + //nolint:gosec // G404: seeded math/rand keeps the test deterministic + rnd := rand.New(rand.NewSource(1)) + for i := 0; i < 10; i++ { + if r, ok := store.RandomOpen(rnd, t0.Add(time.Minute)); !ok || r.OCID != "new" { + t.Fatalf("RandomOpen = %s, %v; want new", r.OCID, ok) + } + } + if _, ok := store.RandomOpen(rnd, t0.Add(2*time.Hour)); ok { + t.Error("everything is stale: expected none") + } +} diff --git a/internal/util/worker_test.go b/internal/util/worker_test.go new file mode 100644 index 0000000..e194208 --- /dev/null +++ b/internal/util/worker_test.go @@ -0,0 +1,252 @@ +package util + +import ( + "context" + "errors" + "fmt" + "io" + "log/slog" + "os" + "sync" + "testing" + "time" +) + +func TestMain(m *testing.M) { + // Keep worker log lines out of the test output. + slog.SetDefault(slog.New(slog.NewTextHandler(io.Discard, nil))) + os.Exit(m.Run()) +} + +func TestWorkerRunRejectsBadConfig(t *testing.T) { + noop := func(context.Context, time.Time) error { return nil } + tests := []struct { + name string + w Worker + }{ + {"no task", Worker{Name: "a", Interval: time.Second}}, + {"zero interval", Worker{Name: "b", Task: noop}}, + {"negative interval", Worker{Name: "c", Interval: -time.Second, Task: noop}}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + if err := tc.w.Run(context.Background(), time.Time{}); err == nil { + t.Fatal("expected an error") + } + }) + } +} + +func TestWorkerRunCancelledBeforeStart(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + called := false + w := Worker{Name: "w", Interval: time.Millisecond, Task: func(context.Context, time.Time) error { + called = true + return nil + }} + if err := w.Run(ctx, time.Time{}); err != nil { + t.Fatal(err) + } + if called { + t.Fatal("task ran after the context was already cancelled") + } +} + +// The checkpoint passed to each run advances only when the previous run +// succeeded or reported ErrWorkerNoWork. +func TestWorkerRunCheckpoints(t *testing.T) { + errFail := errors.New("boom") + results := []error{ + nil, // advance + errFail, // keep + context.Canceled, // keep + fmt.Errorf("%w: x", ErrWorkerNoWork), // advance + nil, + } + since0 := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + var ( + mu sync.Mutex + seen []time.Time + starts []time.Time + ) + w := Worker{ + Name: "w", + Interval: 5 * time.Millisecond, + Task: func(_ context.Context, since time.Time) error { + mu.Lock() + defer mu.Unlock() + i := len(seen) + seen = append(seen, since) + starts = append(starts, time.Now()) + if i == len(results)-1 { + cancel() + } + return results[i] + }, + } + + done := make(chan error, 1) + go func() { done <- w.Run(ctx, since0) }() + select { + case err := <-done: + if err != nil { + t.Fatal(err) + } + case <-time.After(5 * time.Second): + t.Fatal("worker did not stop") + } + + mu.Lock() + defer mu.Unlock() + if len(seen) != len(results) { + t.Fatalf("task ran %d times, want %d", len(seen), len(results)) + } + if !seen[0].Equal(since0) { + t.Fatalf("first run since = %v, want %v", seen[0], since0) + } + // Run 0 succeeded, so runs 1..3 see a checkpoint from run 0's start. + for i := 1; i <= 3; i++ { + if !seen[i].Equal(seen[1]) { + t.Fatalf("run %d since = %v, want unchanged %v", i, seen[i], seen[1]) + } + } + if !seen[1].After(since0) || seen[1].After(starts[1]) { + t.Fatalf("checkpoint after success = %v, want between %v and %v", seen[1], since0, starts[1]) + } + // Run 3 returned ErrWorkerNoWork, so run 4 sees a newer checkpoint. + if !seen[4].After(seen[3]) { + t.Fatalf("checkpoint did not advance after ErrWorkerNoWork: %v -> %v", seen[3], seen[4]) + } +} + +// An in-flight task keeps a live context after the parent is cancelled and is +// allowed to finish. +func TestWorkerRunGracefulShutdown(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + var workErr error + w := Worker{ + Name: "w", Interval: time.Hour, ShutdownTimeout: time.Second, + Task: func(taskCtx context.Context, _ time.Time) error { + cancel() + time.Sleep(10 * time.Millisecond) + workErr = taskCtx.Err() + return nil + }, + } + if err := w.Run(ctx, time.Time{}); err != nil { + t.Fatal(err) + } + if workErr != nil { + t.Fatalf("task context cancelled before shutdown timeout: %v", workErr) + } +} + +// A task that outlives ShutdownTimeout has its context cancelled. +func TestWorkerRunShutdownTimeout(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + w := Worker{ + Name: "w", Interval: time.Hour, ShutdownTimeout: 20 * time.Millisecond, + Task: func(taskCtx context.Context, _ time.Time) error { + cancel() + select { + case <-taskCtx.Done(): + return taskCtx.Err() + case <-time.After(5 * time.Second): + return errors.New("task context never cancelled") + } + }, + } + start := time.Now() + if err := w.Run(ctx, time.Time{}); err != nil { + t.Fatal(err) + } + if d := time.Since(start); d > 2*time.Second { + t.Fatalf("shutdown took %v", d) + } +} + +func TestWorkerExecute(t *testing.T) { + since := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + logger := slog.Default() + tests := []struct { + name string + err error + advance bool + }{ + {"success", nil, true}, + {"no work", ErrWorkerNoWork, true}, + {"wrapped no work", fmt.Errorf("%w: nothing", ErrWorkerNoWork), true}, + {"cancelled", context.Canceled, false}, + {"wrapped cancelled", fmt.Errorf("x: %w", context.Canceled), false}, + {"failure", errors.New("boom"), false}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + w := Worker{Task: func(context.Context, time.Time) error { return tc.err }} + before := time.Now() + got := w.execute(context.Background(), logger, since) + if tc.advance { + if got.Before(before) { + t.Fatalf("checkpoint %v not advanced to run start (>= %v)", got, before) + } + } else if !got.Equal(since) { + t.Fatalf("checkpoint = %v, want unchanged %v", got, since) + } + }) + } +} + +func TestAwaitTask(t *testing.T) { + errFail := errors.New("boom") + v, err := AwaitTask(context.Background(), func() (int, error) { return 42, nil }) + if v != 42 || err != nil { + t.Fatalf("got (%d, %v), want (42, nil)", v, err) + } + v, err = AwaitTask(context.Background(), func() (int, error) { return 7, errFail }) + if v != 7 || !errors.Is(err, errFail) { + t.Fatalf("got (%d, %v), want (7, %v)", v, err, errFail) + } + + ctx, cancel := context.WithCancel(context.Background()) + release := make(chan struct{}) + defer close(release) + go func() { time.Sleep(5 * time.Millisecond); cancel() }() + v, err = AwaitTask(ctx, func() (int, error) { <-release; return 1, nil }) + if v != 0 || !errors.Is(err, context.Canceled) { + t.Fatalf("got (%d, %v), want (0, context.Canceled)", v, err) + } +} + +func TestAwaitTaskE(t *testing.T) { + errFail := errors.New("boom") + if err := AwaitTaskE(context.Background(), func() error { return nil }); err != nil { + t.Fatal(err) + } + if err := AwaitTaskE( + context.Background(), + func() error { return errFail }, + ); !errors.Is( + err, + errFail, + ) { + t.Fatalf("got %v, want %v", err, errFail) + } + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Millisecond) + defer cancel() + release := make(chan struct{}) + defer close(release) + if err := AwaitTaskE( + ctx, + func() error { <-release; return nil }, + ); !errors.Is( + err, + context.DeadlineExceeded, + ) { + t.Fatalf("got %v, want context.DeadlineExceeded", err) + } +}