diff --git a/README.md b/README.md index d6712d0..0c70f9c 100644 --- a/README.md +++ b/README.md @@ -57,8 +57,6 @@ Global flags: `--config`/`-c` (config file), `--debug`/`-d` and | Command | Purpose | |---|---| -| `job-search fetch service` | Poll every enabled source on its interval until stopped (Ctrl+C or SIGTERM) | -| `job-search fetch once` | Fetch from Find a Tender once and print the releases as JSON | | `job-search config init [path]` | Create a config file | | `job-search config show` | Print the configuration (`--json` or `--yaml` for those formats) | | `job-search config get ` | Print one value, e.g. `logging.level` | @@ -70,17 +68,28 @@ Global flags: `--config`/`-c` (config file), `--debug`/`-d` and | `job-search migrate status` | Show the database version and which migrations are applied or pending | | `job-search version` | Print version information | -`fetch service` logs to `logging.file` if it's set, or to the terminal with -`--foreground`. Both fetch commands take `--since "YYYY-MM-DD[ hh:mm:ss]"`. -`fetch once` can be limited with `--planning`, `--opportunities` or -`--awards`, and with `--ignore-rate-limit` it fails on a rate limit instead -of waiting. - The `migrate` commands work on the database in `database.file`. The store refuses to open a database that isn't at this build's schema version, so run `job-search migrate` after installing a new version. +## Fetch tool +`fetcher` reads the same configuration as `job-search`: + +| Command | Purpose | +|---|---| +| `fetcher service` | Poll every enabled source on its interval until stopped (Ctrl+C or SIGTERM) | +| `fetcher fts-once` | Fetch from Find a Tender once and print the releases as JSON | + +`service` logs to `logging.file` if it's set, or to the terminal with +`--foreground`. Both commands take `--since "YYYY-MM-DD[ hh:mm:ss]"`. +`fts-once` can be limited with `--planning`, `--opportunities` or +`--awards`, and with `--ignore-rate-limit` it fails on a rate limit instead +of waiting. + +During development, `go run ./cmd/fetcher` works as well. + + ## Mock Find a Tender server To develop without using the real API, run the mock and enable the `fts_mock` source: @@ -115,6 +124,7 @@ check), `--url` and `--timeout`; see `ws-test dial --help`. | Path | Contents | |---|---| | `cmd/jobsearch` | The `job-search` command | +| `cmd/fetcher` | The `fetcher` command (Find a Tender fetch, the polling service) | | `cmd/mock-fts` | The mock FTS server | | `cmd/ws-test` | The `ws-test` WebSocket test tool | | `internal/cli` | Cobra commands and shared CLI setup | diff --git a/cmd/fetcher/main.go b/cmd/fetcher/main.go new file mode 100644 index 0000000..d5a719b --- /dev/null +++ b/cmd/fetcher/main.go @@ -0,0 +1,82 @@ +/* +Copyright © 2026 Craig Williams + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Lesser General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Lesser General Public License for more details. + +You should have received a copy of the GNU Lesser General Public License +along with this program. If not, see . +*/ +// Command fetcher, for fetching opportunities from JobSearch's sources. Run +// `fetcher --help` for usage. +package main + +import ( + "fmt" + "log/slog" + "os" + "strings" + + "github.com/spf13/cobra" + + "git.avsdev.uk/Go/job-search/internal/cli" + "git.avsdev.uk/Go/job-search/internal/cli/fetch" +) + +// newRootCmd builds the command tree. It returns a fresh tree on each call so +// tests can run commands in isolation. +func newRootCmd() *cobra.Command { + opts := &cli.Options{} + + root := &cobra.Command{ + Use: "fetcher", + Short: "Fetch tools for JobSearch", + Long: `fetcher fetches from Find a Tender once and prints the releases as +JSON (fts-once), or polls every enabled source on its interval until stopped +(service). It reads the JobSearch configuration. + `, + SilenceUsage: true, // don't print usage on runtime errors + SilenceErrors: true, // main prints the error once + CompletionOptions: cobra.CompletionOptions{ + HiddenDefaultCmd: true, + }, + PersistentPreRunE: func(cmd *cobra.Command, args []string) error { + cli.SetupInitialLogging(cmd) + + slog.Debug("command persistent pre-run", "command", "root") + slog.Debug( + "command", + "path", strings.Join(strings.Split(cmd.CommandPath(), " ")[1:], " "), + ) + + return nil + }, + PersistentPostRunE: func(cmd *cobra.Command, args []string) error { + slog.Debug("command persistent post-run", "command", "root") + + return opts.Finish() + }, + } + + cobra.EnableTraverseRunHooks = true + + opts.Bind(root) + + root.AddCommand(fetch.NewCommands(opts)...) + + return root +} + +func main() { + if err := newRootCmd().Execute(); err != nil { + fmt.Fprintln(os.Stderr, "fetcher:", err) + os.Exit(1) + } +} diff --git a/cmd/fetcher/main_test.go b/cmd/fetcher/main_test.go new file mode 100644 index 0000000..7e2c14e --- /dev/null +++ b/cmd/fetcher/main_test.go @@ -0,0 +1,30 @@ +package main + +import ( + "slices" + "testing" +) + +// The commands are tested in internal/cli/fetch. This test checks newRootCmd +// puts them together. + +func TestCommandTree(t *testing.T) { + root := newRootCmd() + + var names []string + for _, c := range root.Commands() { + names = append(names, c.Name()) + } + + for _, want := range []string{"fts-once", "service"} { + if !slices.Contains(names, want) { + t.Errorf("fetcher has no %q command; has %v", want, names) + } + } + + for _, flag := range []string{"config", "debug", "quiet"} { + if root.PersistentFlags().Lookup(flag) == nil { + t.Errorf("fetcher has no --%s flag", flag) + } + } +} diff --git a/cmd/jobsearch/main.go b/cmd/jobsearch/main.go index 20056ec..0e7a498 100644 --- a/cmd/jobsearch/main.go +++ b/cmd/jobsearch/main.go @@ -27,7 +27,6 @@ import ( "git.avsdev.uk/Go/job-search/internal/cli" "git.avsdev.uk/Go/job-search/internal/cli/config" - "git.avsdev.uk/Go/job-search/internal/cli/fetch" "git.avsdev.uk/Go/job-search/internal/cli/migrate" "git.avsdev.uk/Go/job-search/internal/cli/serve" "git.avsdev.uk/Go/job-search/internal/cli/version" @@ -74,7 +73,6 @@ opportunities. root.AddCommand( version.NewCommand(), config.NewCommand(opts), - fetch.NewCommand(opts), migrate.NewCommand(opts), serve.NewCommand(opts), ) diff --git a/cmd/jobsearch/main_test.go b/cmd/jobsearch/main_test.go index 6493ae4..865f1b6 100644 --- a/cmd/jobsearch/main_test.go +++ b/cmd/jobsearch/main_test.go @@ -18,7 +18,7 @@ func TestCommandTree(t *testing.T) { names = append(names, c.Name()) } - commands := []string{"config", "fetch", "migrate", "serve", "version"} + commands := []string{"config", "migrate", "serve", "version"} for _, want := range commands { if !slices.Contains(names, want) { t.Errorf("job-search has no %q command; has %v", want, names) diff --git a/internal/cli/fetch/command.go b/internal/cli/fetch/command.go index 6b801cb..2a383b2 100644 --- a/internal/cli/fetch/command.go +++ b/internal/cli/fetch/command.go @@ -1,29 +1,18 @@ +// Package fetch holds the fetcher commands: "fts-once" fetches from Find a +// Tender once and prints the releases, and "service" polls every enabled +// source on its interval. package fetch import ( - "log/slog" - "github.com/spf13/cobra" "git.avsdev.uk/Go/job-search/internal/cli" ) -// NewCommand returns the "fetch" command, which fetches opportunities once or -// as a service. -func NewCommand(opts *cli.Options) *cobra.Command { - cmd := &cobra.Command{ - Use: "fetch", - Short: "Fetch current opportunities", - PersistentPreRun: func(cmd *cobra.Command, args []string) { - slog.Debug("command persistent pre-run", "command", "fetch") - }, - PersistentPostRun: func(cmd *cobra.Command, args []string) { - slog.Debug("command persistent post-run", "command", "fetch") - }, +// NewCommands returns the fetcher commands, "fts-once" and "service". +func NewCommands(opts *cli.Options) []*cobra.Command { + return []*cobra.Command{ + commandFTSOnce(opts), + commandService(opts), } - - cmd.AddCommand(commandOnce(opts)) - cmd.AddCommand(commandService(opts)) - - return cmd } diff --git a/internal/cli/fetch/once.go b/internal/cli/fetch/fts_once.go similarity index 88% rename from internal/cli/fetch/once.go rename to internal/cli/fetch/fts_once.go index be36247..885e3dd 100644 --- a/internal/cli/fetch/once.go +++ b/internal/cli/fetch/fts_once.go @@ -51,18 +51,20 @@ func filter( return matching } -func commandOnce(opts *cli.Options) *cobra.Command { +// commandFTSOnce returns the "fts-once" command, which fetches from Find a +// Tender once and prints the releases as JSON. +func commandFTSOnce(opts *cli.Options) *cobra.Command { cmd := &cobra.Command{ - Use: "once", - Short: "Fetch current opportunities - single shot", + Use: "fts-once", + Short: "Fetch from Find a Tender once and print the releases as JSON", PersistentPreRun: func(cmd *cobra.Command, args []string) { - slog.Debug("command persistent pre-run", "command", "fetch.once") + slog.Debug("command persistent pre-run", "command", "fts-once") }, PersistentPostRun: func(cmd *cobra.Command, args []string) { - slog.Debug("command persistent post-run", "command", "fetch.once") + slog.Debug("command persistent post-run", "command", "fts-once") }, RunE: func(cmd *cobra.Command, args []string) error { - slog.Debug("command run", "command", "fetch.once") + slog.Debug("command run", "command", "fts-once") conf, err := opts.RunSetup(cmd, true, true) if err != nil { diff --git a/internal/cli/fetch/once_test.go b/internal/cli/fetch/fts_once_test.go similarity index 97% rename from internal/cli/fetch/once_test.go rename to internal/cli/fetch/fts_once_test.go index ad0c1b8..96cfdc9 100644 --- a/internal/cli/fetch/once_test.go +++ b/internal/cli/fetch/fts_once_test.go @@ -54,7 +54,7 @@ func TestFilter(t *testing.T) { // Conflicting filters are rejected by cobra before anything is fetched. func TestOnceFiltersExclusive(t *testing.T) { - cmd := commandOnce(&cli.Options{}) + cmd := commandFTSOnce(&cli.Options{}) cmd.SetArgs([]string{"--planning", "--awards"}) cmd.SetOut(io.Discard) cmd.SetErr(io.Discard) diff --git a/internal/cli/fetch/service.go b/internal/cli/fetch/service.go index 40c7d2e..b3c0339 100644 --- a/internal/cli/fetch/service.go +++ b/internal/cli/fetch/service.go @@ -18,15 +18,15 @@ import ( func commandService(opts *cli.Options) *cobra.Command { cmd := &cobra.Command{ Use: "service", - Short: "Fetch current opportunities - background service", + Short: "Run every enabled source's fetcher on its interval until stopped", PersistentPreRun: func(cmd *cobra.Command, args []string) { - slog.Debug("command persistent pre-run", "command", "fetch.service") + slog.Debug("command persistent pre-run", "command", "service") }, PersistentPostRun: func(cmd *cobra.Command, args []string) { - slog.Debug("command persistent post-run", "command", "fetch.service") + slog.Debug("command persistent post-run", "command", "service") }, RunE: func(cmd *cobra.Command, args []string) error { - slog.Debug("command run", "command", "fetch.service") + slog.Debug("command run", "command", "service") foreground, _ := cmd.Flags().GetBool("foreground") diff --git a/internal/config/README.md b/internal/config/README.md index 6b93b6d..11fd64f 100644 --- a/internal/config/README.md +++ b/internal/config/README.md @@ -42,7 +42,7 @@ Configuration is managed in sections: | Key | Type | Default | Notes | |---|---|---|---| | `logging.level` | string | `warn` | One of `debug`, `info`, `warn`, `error` | -| `logging.file` | string | | Log file (JSON lines) for long-running commands such as `fetch service`. Empty: log to the terminal | +| `logging.file` | string | | Log file (JSON lines) for long-running commands such as `fetcher service`. Empty: log to the terminal | Logging always starts on the terminal. Long-running commands switch to the file in `logging.file` once the configuration is loaded, unless it is empty (they @@ -69,7 +69,7 @@ supported. |---|---|---|---| | `sources..enabled` | bool | `false` | For every source | | `sources..interval` | duration | `10m` | For every source; e.g. `30s`, `1h`. Must be greater than 0; turn a source off with `enabled` | -| `sources.find_a_tender.api_root` | string | `https://www.find-tender.service.gov.uk/api/1.0` | URL of the Find a Tender API. Empty: the source fails to start if enabled, and `fetch once` fails | +| `sources.find_a_tender.api_root` | string | `https://www.find-tender.service.gov.uk/api/1.0` | URL of the Find a Tender API. Empty: the source fails to start if enabled, and `fetcher fts-once` fails | | `sources.fts_mock.api_root` | string | | URL of the mock FTS server (see `internal/mock`). Empty: the source fails to start if enabled | | `sources.sleeper.block_shutdown` | bool | `false` | Test source: if true, its sleep ignores shutdown | diff --git a/internal/config/conf_source_findatender.go b/internal/config/conf_source_findatender.go index da6f239..f5dcc35 100644 --- a/internal/config/conf_source_findatender.go +++ b/internal/config/conf_source_findatender.go @@ -7,7 +7,7 @@ const JSCKSourceFindATender string = "find_a_tender" // JSCSourceFindATender configures fetching from the Find a Tender // OCDS API at APIRoot. Empty: the source fails to start if it is enabled, -// and `fetch once` fails. +// and `fetcher fts-once` fails. type JSCSourceFindATender struct { JSCSourcesSource APIRoot string `cfg:"api_root" default:"https://www.find-tender.service.gov.uk/api/1.0"`