Made the fetcher a Fetcher type with per-source config, and ran it in job-search serve

fetcher.Run(ctx, conf, since) becomes New(&conf.Sources, opts...) and
Run(ctx, since). Each worker now takes only its own source's config, and
Run starts each one through a closure that binds it. Cancelling ctx stops
the workers through the Fetcher's background context, and Run always waits
for every worker before returning their joined errors.

The findatender client no longer holds the configuration: New takes only
the API root, ReloadConfig is gone, and request tracing follows
logging.IsDebugEnabled(). The FindAOpportunity worker is now FindATender.

job-search serve runs the fetcher alongside the HTTP server. A source that
fails stops the server, and the server stopping stops the sources; with no
sources enabled the server keeps running. fetcher service and fetcher
fts-once use the new APIs.
This commit is contained in:
bwilliams committed 2026-10-16 10:44:38 +01:00
1 parent f0c171c8a8
commit 18db462d89
12 files changed
+269 -214

No files matched your search

+1 -1
View File
@@ -79,7 +79,7 @@ func commandFTSOnce(opts *cli.Options) *cobra.Command {
if conf.Sources.FindATender.APIRoot == "" {
return errors.New("sources.find_a_tender.api_root is not set")
}
findATender := findatender.New(conf, conf.Sources.FindATender.APIRoot)
findATender := findatender.New(conf.Sources.FindATender.APIRoot)
defer findATender.Shutdown()
ignoreRateLimit, _ := cmd.Flags().GetBool("ignore-rate-limit")
+1 -1
View File
@@ -53,7 +53,7 @@ func commandService(opts *cli.Options) *cobra.Command {
stop()
}()
return fetcher.Run(ctx, conf, since)
return fetcher.New(&conf.Sources).Run(ctx, since)
},
}
cmd.Flags().Time(
+31 -6
View File
@@ -11,22 +11,27 @@ import (
"strconv"
"strings"
"syscall"
"time"
"github.com/spf13/cobra"
"git.avsdev.uk/Go/job-search/internal/cli"
"git.avsdev.uk/Go/job-search/internal/fetcher"
"git.avsdev.uk/Go/job-search/internal/logging"
"git.avsdev.uk/Go/job-search/internal/server"
)
// NewCommand returns the serve command, which serves the frontend, the REST
// API and the WebSocket endpoint.
// NewCommand returns the serve command, which serves the REST API and the
// WebSocket endpoint, and runs a worker for every enabled source.
func NewCommand(opts *cli.Options) *cobra.Command {
cmd := &cobra.Command{
Use: "serve",
Short: "Serve the frontend (/), REST API (/api) and WebSocket (/ws)",
Long: `Serve the embedded Vite frontend, the REST API under /api and the
WebSocket endpoint under /ws.`,
Short: "Serve the REST API (/api) and WebSocket (/ws), and fetch from sources",
Long: `Serve the REST API under /api and the WebSocket endpoint under /ws.
It also runs a worker for every enabled source, as fetcher service does. A
source that stops with an error stops the server too, and the server
stopping stops the sources.`,
PersistentPreRun: func(cmd *cobra.Command, args []string) {
slog.Debug("command persistent pre-run", "command", "serve")
},
@@ -108,7 +113,27 @@ WebSocket endpoint under /ws.`,
return err
}
return appServer.Run(ctx)
// runCtx stops whichever of the server and the sources is still
// running once the other has stopped.
runCtx, cancelRun := context.WithCancel(ctx)
defer cancelRun()
sourceFetcher := fetcher.New(&conf.Sources)
fetcherDone := make(chan error, 1)
go func() {
fetchErr := sourceFetcher.Run(runCtx, time.Time{})
// Only a failure stops the server: with no sources enabled, Run
// returns nil straight away.
if fetchErr != nil {
cancelRun()
}
fetcherDone <- fetchErr
}()
serverErr := appServer.Run(runCtx)
cancelRun()
return errors.Join(serverErr, <-fetcherDone)
},
}