Made the fetcher a Fetcher type with per-source config, and ran it in job-search serve

fetcher.Run(ctx, conf, since) becomes New(&conf.Sources, opts...) and
Run(ctx, since). Each worker now takes only its own source's config, and
Run starts each one through a closure that binds it. Cancelling ctx stops
the workers through the Fetcher's background context, and Run always waits
for every worker before returning their joined errors.

The findatender client no longer holds the configuration: New takes only
the API root, ReloadConfig is gone, and request tracing follows
logging.IsDebugEnabled(). The FindAOpportunity worker is now FindATender.

job-search serve runs the fetcher alongside the HTTP server. A source that
fails stops the server, and the server stopping stops the sources; with no
sources enabled the server keeps running. fetcher service and fetcher
fts-once use the new APIs.

internal/findatender/api.go

@@ -307,10 +307,10 @@ func closeBody(res *resty.Response) {
 }
 
 // fetchURI performs a single GET of uri, using findATender's context, and
-// traces it at debug level. Unlike the method of the same name it does not
-// check the response status.
+// traces it when debug logging is on (logging.IsDebugEnabled). Unlike the
+// method of the same name it does not check the response status.
 func fetchURI(findATender *FindATender, uri string) (*resty.Response, error) {
-	withTrace := findATender.config.Logging.LogLevel() <= slog.LevelDebug
+	withTrace := logging.IsDebugEnabled()
 	logger := findATender.logger()
 
 	logger.Info("fetching", "uri", uri)

internal/findatender/api_test.go

@@ -17,18 +17,10 @@ import (
 
 	resty "resty.dev/v3"
 
-	"git.avsdev.uk/Go/job-search/internal/config"
 	"git.avsdev.uk/Go/job-search/internal/findatender/ocds"
+	"git.avsdev.uk/Go/job-search/internal/logging"
 )
 
-func testConfig(level string) *config.JobSearchConfig {
-	return &config.JobSearchConfig{
-		Logging: config.JSCLogging{
-			Level: level,
-		},
-	}
-}
-
 // quietLogs discards slog output for the duration of the test, or sends it
 // through h when given.
 func quietLogs(t *testing.T, h slog.Handler) {
@@ -44,16 +36,24 @@ func quietLogs(t *testing.T, h slog.Handler) {
 }
 
 // newTestFT starts h on a test server and returns a FindATender pointed at it.
+// With debug, the logging level is debug for the test, so requests are
+// traced.
 func newTestFT(
-	t *testing.T, level string, h http.HandlerFunc,
+	t *testing.T, debug bool, h http.HandlerFunc,
 ) (*FindATender, *httptest.Server) {
 	t.Helper()
 	quietLogs(t, nil)
+	if debug {
+		logging.SetLevel(slog.LevelDebug)
+		t.Cleanup(func() {
+			logging.SetLevel(slog.LevelInfo)
+		})
+	}
 
 	srv := httptest.NewServer(h)
 	t.Cleanup(srv.Close)
 
-	ft := New(testConfig(level), srv.URL)
+	ft := New(srv.URL)
 	t.Cleanup(ft.Shutdown)
 
 	return ft, srv
@@ -97,7 +97,7 @@ func TestRateSleep(t *testing.T) {
 
 func TestFetchWithOptsBuildsURI(t *testing.T) {
 	var gotPath, gotQuery string
-	ft, _ := newTestFT(t, "warn", func(w http.ResponseWriter, r *http.Request) {
+	ft, _ := newTestFT(t, false, func(w http.ResponseWriter, r *http.Request) {
 		gotPath, gotQuery = r.URL.Path, r.URL.RawQuery
 		io.WriteString(w, packageJSON(""))
 	})
@@ -133,7 +133,7 @@ func TestFetchWithOptsBuildsURI(t *testing.T) {
 // A zero Since defaults to 24 hours ago.
 func TestFetchWithOptsDefaultSince(t *testing.T) {
 	var gotQuery string
-	ft, _ := newTestFT(t, "warn", func(w http.ResponseWriter, r *http.Request) {
+	ft, _ := newTestFT(t, false, func(w http.ResponseWriter, r *http.Request) {
 		gotQuery = r.URL.Query().Get("updatedFrom")
 		io.WriteString(w, packageJSON(""))
 	})
@@ -188,7 +188,7 @@ func TestFetchURIRateLimit(t *testing.T) {
 			var hits atomic.Int32
 			ft, srv := newTestFT(
 				t,
-				"warn",
+				false,
 				func(w http.ResponseWriter, r *http.Request) {
 					if hits.Add(1) == 1 {
 						if tt.retryAfter != "" {
@@ -228,7 +228,7 @@ func TestFetchURIRateLimit(t *testing.T) {
 }
 
 func TestFetchURIRateLimitCancelled(t *testing.T) {
-	ft, srv := newTestFT(t, "warn", func(w http.ResponseWriter, r *http.Request) {
+	ft, srv := newTestFT(t, false, func(w http.ResponseWriter, r *http.Request) {
 		w.Header().Set("Retry-After", "3600")
 		w.WriteHeader(http.StatusTooManyRequests)
 	})
@@ -252,7 +252,7 @@ func TestFetchURIRateLimitCancelled(t *testing.T) {
 func TestFetchURIRateLimitTwice(t *testing.T) {
 	// A second 429 after waiting is reported as ErrRateLimitExceeded.
 	var hits atomic.Int32
-	ft, srv := newTestFT(t, "warn", func(w http.ResponseWriter, r *http.Request) {
+	ft, srv := newTestFT(t, false, func(w http.ResponseWriter, r *http.Request) {
 		hits.Add(1)
 		w.Header().Set("Retry-After", "0")
 		w.WriteHeader(http.StatusTooManyRequests)
@@ -274,7 +274,7 @@ func TestFetchURIUnexpectedStatus(t *testing.T) {
 		t.Run(http.StatusText(status), func(t *testing.T) {
 			ft, srv := newTestFT(
 				t,
-				"warn",
+				false,
 				func(w http.ResponseWriter, r *http.Request) {
 					w.WriteHeader(status)
 					io.WriteString(w, packageJSON("", releaseJSON("ocds-1", "1")))
@@ -294,7 +294,7 @@ func TestFetchURIUnexpectedStatus(t *testing.T) {
 // A non-2xx after waiting out a 429 is reported as ErrUnexpectedStatus.
 func TestFetchURIRateLimitThenError(t *testing.T) {
 	var hits atomic.Int32
-	ft, srv := newTestFT(t, "warn", func(w http.ResponseWriter, r *http.Request) {
+	ft, srv := newTestFT(t, false, func(w http.ResponseWriter, r *http.Request) {
 		if hits.Add(1) == 1 {
 			w.Header().Set("Retry-After", "0")
 			w.WriteHeader(http.StatusTooManyRequests)
@@ -309,7 +309,7 @@ func TestFetchURIRateLimitThenError(t *testing.T) {
 }
 
 func TestFetchURIConnectionError(t *testing.T) {
-	ft, srv := newTestFT(t, "warn", func(w http.ResponseWriter, r *http.Request) {
+	ft, srv := newTestFT(t, false, func(w http.ResponseWriter, r *http.Request) {
 	})
 	srv.Close()
 	ft.httpClient.SetRetryCount(0)
@@ -319,7 +319,7 @@ func TestFetchURIConnectionError(t *testing.T) {
 }
 
 func TestFetchURICancelledContext(t *testing.T) {
-	ft, srv := newTestFT(t, "warn", func(w http.ResponseWriter, r *http.Request) {
+	ft, srv := newTestFT(t, false, func(w http.ResponseWriter, r *http.Request) {
 		io.WriteString(w, packageJSON(""))
 	})
 	ctx, cancel := context.WithCancel(context.Background())
@@ -346,7 +346,7 @@ func TestFetchURIDebugTrace(t *testing.T) {
 		t.Run(name, func(t *testing.T) {
 			ft, _ := newTestFT(
 				t,
-				"debug",
+				true,
 				func(w http.ResponseWriter, r *http.Request) {
 					io.WriteString(w, packageJSON("", releaseJSON("ocds-1", "1")))
 				},
@@ -379,7 +379,7 @@ func TestFetchURIDebugTrace(t *testing.T) {
 func TestFetchURIDebugTraceConnectionError(t *testing.T) {
 	ft, srv := newTestFT(
 		t,
-		"debug",
+		true,
 		func(w http.ResponseWriter, r *http.Request) {
 		},
 	)
@@ -392,7 +392,7 @@ func TestFetchURIDebugTraceConnectionError(t *testing.T) {
 
 func TestFetchNext(t *testing.T) {
 	var gotPath string
-	ft, srv := newTestFT(t, "warn", func(w http.ResponseWriter, r *http.Request) {
+	ft, srv := newTestFT(t, false, func(w http.ResponseWriter, r *http.Request) {
 		gotPath = r.URL.Path
 		io.WriteString(w, packageJSON(""))
 	})
@@ -428,7 +428,7 @@ func TestFetchNext(t *testing.T) {
 // responseFor fetches body from a test server so it can be unpacked.
 func responseFor(t *testing.T, body string) *resty.Response {
 	t.Helper()
-	ft, srv := newTestFT(t, "warn", func(w http.ResponseWriter, r *http.Request) {
+	ft, srv := newTestFT(t, false, func(w http.ResponseWriter, r *http.Request) {
 		io.WriteString(w, body)
 	})
 	res, err := ft.fetchURI(srv.URL, false)
@@ -551,7 +551,7 @@ func pagedServer(
 	t.Helper()
 	var hits atomic.Int32
 	var srvURL string
-	ft, srv := newTestFT(t, "warn", func(w http.ResponseWriter, r *http.Request) {
+	ft, srv := newTestFT(t, false, func(w http.ResponseWriter, r *http.Request) {
 		hits.Add(1)
 
 		i := 0
@@ -598,7 +598,7 @@ func TestFetchOnceErrors(t *testing.T) {
 		t.Errorf("empty page: err = %v, want ErrNoReleases", err)
 	}
 
-	ft, _ = newTestFT(t, "warn", func(w http.ResponseWriter, r *http.Request) {
+	ft, _ = newTestFT(t, false, func(w http.ResponseWriter, r *http.Request) {
 		io.WriteString(w, "not json")
 	})
 	if _, err := ft.fetchOnce(RequestOptions{
@@ -607,7 +607,7 @@ func TestFetchOnceErrors(t *testing.T) {
 		t.Error("bad body: expected an error")
 	}
 
-	ft, _ = newTestFT(t, "warn", func(w http.ResponseWriter, r *http.Request) {
+	ft, _ = newTestFT(t, false, func(w http.ResponseWriter, r *http.Request) {
 		w.WriteHeader(http.StatusTooManyRequests)
 	})
 	if _, err := ft.fetchOnce(
@@ -655,7 +655,7 @@ func TestFetchAllErrors(t *testing.T) {
 	}
 
 	// First page fails.
-	ft, _ = newTestFT(t, "warn", func(w http.ResponseWriter, r *http.Request) {
+	ft, _ = newTestFT(t, false, func(w http.ResponseWriter, r *http.Request) {
 		w.WriteHeader(http.StatusTooManyRequests)
 	})
 	if _, err := ft.fetchAll(
@@ -666,7 +666,7 @@ func TestFetchAllErrors(t *testing.T) {
 		t.Errorf("first page rate limited: err = %v", err)
 	}
 
-	ft, _ = newTestFT(t, "warn", func(w http.ResponseWriter, r *http.Request) {
+	ft, _ = newTestFT(t, false, func(w http.ResponseWriter, r *http.Request) {
 		io.WriteString(w, "not json")
 	})
 	if _, err := ft.fetchAll(RequestOptions{
@@ -688,7 +688,7 @@ func TestFetchAllErrors(t *testing.T) {
 			var srvURL string
 			ft, srv := newTestFT(
 				t,
-				"warn",
+				false,
 				func(w http.ResponseWriter, r *http.Request) {
 					if r.URL.Query().Get("page") == "" {
 						io.WriteString(
@@ -730,7 +730,7 @@ func TestFetchAllRepeatedNextLink(t *testing.T) {
 			var srvURL string
 			ft, srv := newTestFT(
 				t,
-				"error",
+				false,
 				func(w http.ResponseWriter, r *http.Request) {
 					if hits.Add(1) > 10 {
 						t.Error("fetchAll did not stop following next links")

internal/findatender/findatender.go

@@ -7,7 +7,6 @@ import (
 
 	resty "resty.dev/v3"
 
-	"git.avsdev.uk/Go/job-search/internal/config"
 	"git.avsdev.uk/Go/job-search/internal/types"
 )
 
@@ -15,7 +14,6 @@ import (
 // server). Create it with New and call Shutdown when finished with it.
 type FindATender struct {
 	httpClient *resty.Client
-	config     *config.JobSearchConfig
 	apiRoot    string
 	ctx        context.Context
 	log        *slog.Logger
@@ -34,11 +32,8 @@ func WithLogger(logger *slog.Logger) Option {
 // New returns a client for the OCDS API at apiRoot. Requests time out after
 // 60 seconds and are retried once. Call Shutdown when finished with it. It
 // logs to slog.Default() unless given WithLogger.
-func New(
-	conf *config.JobSearchConfig, apiRoot string, opts ...Option,
-) *FindATender {
+func New(apiRoot string, opts ...Option) *FindATender {
 	findATender := &FindATender{
-		config:  conf,
 		apiRoot: apiRoot,
 	}
 	for _, opt := range opts {
@@ -61,12 +56,6 @@ func (ft *FindATender) logger() *slog.Logger {
 	return log.With("component", "findatender")
 }
 
-// ReloadConfig replaces the configuration the client reads, such as the log
-// level. The HTTP client and API root set by New are not changed.
-func (ft *FindATender) ReloadConfig(conf *config.JobSearchConfig) {
-	ft.config = conf
-}
-
 // With returns a copy of the client whose requests use ctx, so they stop
 // when it is cancelled. A nil ctx returns the client unchanged.
 func (ft *FindATender) With(ctx context.Context) *FindATender {

internal/findatender/findatender_test.go

@@ -14,12 +14,11 @@ import (
 )
 
 func TestNew(t *testing.T) {
-	conf := testConfig("warn")
-	ft := New(conf, "https://example/api")
+	ft := New("https://example/api")
 	defer ft.Shutdown()
 
-	if ft.config != conf || ft.apiRoot != "https://example/api" {
-		t.Errorf("New did not keep its arguments: %+v", ft)
+	if ft.apiRoot != "https://example/api" {
+		t.Errorf("New did not keep its API root: %+v", ft)
 	}
 	if ft.httpClient == nil {
 		t.Fatal("no http client")
@@ -37,18 +36,8 @@ func TestNew(t *testing.T) {
 	}
 }
 
-func TestReloadConfig(t *testing.T) {
-	ft := New(testConfig("warn"), "")
-	defer ft.Shutdown()
-	conf := testConfig("debug")
-	ft.ReloadConfig(conf)
-	if ft.config != conf {
-		t.Error("config not replaced")
-	}
-}
-
 func TestWith(t *testing.T) {
-	ft := New(testConfig("warn"), "")
+	ft := New("")
 	defer ft.Shutdown()
 
 	// Passing nil on purpose to test that case.
@@ -69,8 +58,8 @@ func TestWith(t *testing.T) {
 	if ft.ctx != nil {
 		t.Error("With changed the original")
 	}
-	if got.httpClient != ft.httpClient || got.config != ft.config {
-		t.Error("copy should share the client and config")
+	if got.httpClient != ft.httpClient {
+		t.Error("copy should share the client")
 	}
 }
 
@@ -128,7 +117,7 @@ func TestFetchAllPublic(t *testing.T) {
 		}
 	}
 
-	ft, _ = newTestFT(t, "warn", func(w http.ResponseWriter, r *http.Request) {
+	ft, _ = newTestFT(t, false, func(w http.ResponseWriter, r *http.Request) {
 		io.WriteString(w, "not json")
 	})
 	if _, err := ft.FetchAll(time.Now(), false); err == nil {