Go/job-search
1
0 Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases 2 Wiki Activity

Made the fetcher a Fetcher type with per-source config, and ran it in job-search serve

Browse Source 
fetcher.Run(ctx, conf, since) becomes New(&conf.Sources, opts...) and
Run(ctx, since). Each worker now takes only its own source's config, and
Run starts each one through a closure that binds it. Cancelling ctx stops
the workers through the Fetcher's background context, and Run always waits
for every worker before returning their joined errors.

The findatender client no longer holds the configuration: New takes only
the API root, ReloadConfig is gone, and request tracing follows
logging.IsDebugEnabled(). The FindAOpportunity worker is now FindATender.

job-search serve runs the fetcher alongside the HTTP server. A source that
fails stops the server, and the server stopping stops the sources; with no
sources enabled the server keeps running. fetcher service and fetcher
fts-once use the new APIs.
This commit is contained in:
bwilliams committed 2026-10-16 10:44:38 +01:00
1 parent f0c171c8a8
commit 18db462d89
12 files changed
+269 -214
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

internal/findatender/api_test.go
+31 -31
View File
@@ -17,18 +17,10 @@ import (
resty "resty.dev/v3"
"git.avsdev.uk/Go/job-search/internal/config"
"git.avsdev.uk/Go/job-search/internal/findatender/ocds"
"git.avsdev.uk/Go/job-search/internal/logging"
)
func testConfig(level string) *config.JobSearchConfig {
return &config.JobSearchConfig{
Logging: config.JSCLogging{
Level: level,
},
}
}
// quietLogs discards slog output for the duration of the test, or sends it
// through h when given.
func quietLogs(t *testing.T, h slog.Handler) {
@@ -44,16 +36,24 @@ func quietLogs(t *testing.T, h slog.Handler) {
}
// newTestFT starts h on a test server and returns a FindATender pointed at it.
// With debug, the logging level is debug for the test, so requests are
// traced.
func newTestFT(
t *testing.T, level string, h http.HandlerFunc,
t *testing.T, debug bool, h http.HandlerFunc,
) (*FindATender, *httptest.Server) {
t.Helper()
quietLogs(t, nil)
if debug {
logging.SetLevel(slog.LevelDebug)
t.Cleanup(func() {
logging.SetLevel(slog.LevelInfo)
})
}
srv := httptest.NewServer(h)
t.Cleanup(srv.Close)
ft := New(testConfig(level), srv.URL)
ft := New(srv.URL)
t.Cleanup(ft.Shutdown)
return ft, srv
@@ -97,7 +97,7 @@ func TestRateSleep(t *testing.T) {
func TestFetchWithOptsBuildsURI(t *testing.T) {
var gotPath, gotQuery string
ft, _ := newTestFT(t, "warn", func(w http.ResponseWriter, r *http.Request) {
ft, _ := newTestFT(t, false, func(w http.ResponseWriter, r *http.Request) {
gotPath, gotQuery = r.URL.Path, r.URL.RawQuery
io.WriteString(w, packageJSON(""))
})
@@ -133,7 +133,7 @@ func TestFetchWithOptsBuildsURI(t *testing.T) {
// A zero Since defaults to 24 hours ago.
func TestFetchWithOptsDefaultSince(t *testing.T) {
var gotQuery string
ft, _ := newTestFT(t, "warn", func(w http.ResponseWriter, r *http.Request) {
ft, _ := newTestFT(t, false, func(w http.ResponseWriter, r *http.Request) {
gotQuery = r.URL.Query().Get("updatedFrom")
io.WriteString(w, packageJSON(""))
})
@@ -188,7 +188,7 @@ func TestFetchURIRateLimit(t *testing.T) {
var hits atomic.Int32
ft, srv := newTestFT(
t,
"warn",
false,
func(w http.ResponseWriter, r *http.Request) {
if hits.Add(1) == 1 {
if tt.retryAfter != "" {
@@ -228,7 +228,7 @@ func TestFetchURIRateLimit(t *testing.T) {
}
func TestFetchURIRateLimitCancelled(t *testing.T) {
ft, srv := newTestFT(t, "warn", func(w http.ResponseWriter, r *http.Request) {
ft, srv := newTestFT(t, false, func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Retry-After", "3600")
w.WriteHeader(http.StatusTooManyRequests)
})
@@ -252,7 +252,7 @@ func TestFetchURIRateLimitCancelled(t *testing.T) {
func TestFetchURIRateLimitTwice(t *testing.T) {
// A second 429 after waiting is reported as ErrRateLimitExceeded.
var hits atomic.Int32
ft, srv := newTestFT(t, "warn", func(w http.ResponseWriter, r *http.Request) {
ft, srv := newTestFT(t, false, func(w http.ResponseWriter, r *http.Request) {
hits.Add(1)
w.Header().Set("Retry-After", "0")
w.WriteHeader(http.StatusTooManyRequests)
@@ -274,7 +274,7 @@ func TestFetchURIUnexpectedStatus(t *testing.T) {
t.Run(http.StatusText(status), func(t *testing.T) {
ft, srv := newTestFT(
t,
"warn",
false,
func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(status)
io.WriteString(w, packageJSON("", releaseJSON("ocds-1", "1")))
@@ -294,7 +294,7 @@ func TestFetchURIUnexpectedStatus(t *testing.T) {
// A non-2xx after waiting out a 429 is reported as ErrUnexpectedStatus.
func TestFetchURIRateLimitThenError(t *testing.T) {
var hits atomic.Int32
ft, srv := newTestFT(t, "warn", func(w http.ResponseWriter, r *http.Request) {
ft, srv := newTestFT(t, false, func(w http.ResponseWriter, r *http.Request) {
if hits.Add(1) == 1 {
w.Header().Set("Retry-After", "0")
w.WriteHeader(http.StatusTooManyRequests)
@@ -309,7 +309,7 @@ func TestFetchURIRateLimitThenError(t *testing.T) {
}
func TestFetchURIConnectionError(t *testing.T) {
ft, srv := newTestFT(t, "warn", func(w http.ResponseWriter, r *http.Request) {
ft, srv := newTestFT(t, false, func(w http.ResponseWriter, r *http.Request) {
})
srv.Close()
ft.httpClient.SetRetryCount(0)
@@ -319,7 +319,7 @@ func TestFetchURIConnectionError(t *testing.T) {
}
func TestFetchURICancelledContext(t *testing.T) {
ft, srv := newTestFT(t, "warn", func(w http.ResponseWriter, r *http.Request) {
ft, srv := newTestFT(t, false, func(w http.ResponseWriter, r *http.Request) {
io.WriteString(w, packageJSON(""))
})
ctx, cancel := context.WithCancel(context.Background())
@@ -346,7 +346,7 @@ func TestFetchURIDebugTrace(t *testing.T) {
t.Run(name, func(t *testing.T) {
ft, _ := newTestFT(
t,
"debug",
true,
func(w http.ResponseWriter, r *http.Request) {
io.WriteString(w, packageJSON("", releaseJSON("ocds-1", "1")))
},
@@ -379,7 +379,7 @@ func TestFetchURIDebugTrace(t *testing.T) {
func TestFetchURIDebugTraceConnectionError(t *testing.T) {
ft, srv := newTestFT(
t,
"debug",
true,
func(w http.ResponseWriter, r *http.Request) {
},
)
@@ -392,7 +392,7 @@ func TestFetchURIDebugTraceConnectionError(t *testing.T) {
func TestFetchNext(t *testing.T) {
var gotPath string
ft, srv := newTestFT(t, "warn", func(w http.ResponseWriter, r *http.Request) {
ft, srv := newTestFT(t, false, func(w http.ResponseWriter, r *http.Request) {
gotPath = r.URL.Path
io.WriteString(w, packageJSON(""))
})
@@ -428,7 +428,7 @@ func TestFetchNext(t *testing.T) {
// responseFor fetches body from a test server so it can be unpacked.
func responseFor(t *testing.T, body string) *resty.Response {
t.Helper()
ft, srv := newTestFT(t, "warn", func(w http.ResponseWriter, r *http.Request) {
ft, srv := newTestFT(t, false, func(w http.ResponseWriter, r *http.Request) {
io.WriteString(w, body)
})
res, err := ft.fetchURI(srv.URL, false)
@@ -551,7 +551,7 @@ func pagedServer(
t.Helper()
var hits atomic.Int32
var srvURL string
ft, srv := newTestFT(t, "warn", func(w http.ResponseWriter, r *http.Request) {
ft, srv := newTestFT(t, false, func(w http.ResponseWriter, r *http.Request) {
hits.Add(1)
i := 0
@@ -598,7 +598,7 @@ func TestFetchOnceErrors(t *testing.T) {
t.Errorf("empty page: err = %v, want ErrNoReleases", err)
}
ft, _ = newTestFT(t, "warn", func(w http.ResponseWriter, r *http.Request) {
ft, _ = newTestFT(t, false, func(w http.ResponseWriter, r *http.Request) {
io.WriteString(w, "not json")
})
if _, err := ft.fetchOnce(RequestOptions{
@@ -607,7 +607,7 @@ func TestFetchOnceErrors(t *testing.T) {
t.Error("bad body: expected an error")
}
ft, _ = newTestFT(t, "warn", func(w http.ResponseWriter, r *http.Request) {
ft, _ = newTestFT(t, false, func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusTooManyRequests)
})
if _, err := ft.fetchOnce(
@@ -655,7 +655,7 @@ func TestFetchAllErrors(t *testing.T) {
}
// First page fails.
ft, _ = newTestFT(t, "warn", func(w http.ResponseWriter, r *http.Request) {
ft, _ = newTestFT(t, false, func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusTooManyRequests)
})
if _, err := ft.fetchAll(
@@ -666,7 +666,7 @@ func TestFetchAllErrors(t *testing.T) {
t.Errorf("first page rate limited: err = %v", err)
}
ft, _ = newTestFT(t, "warn", func(w http.ResponseWriter, r *http.Request) {
ft, _ = newTestFT(t, false, func(w http.ResponseWriter, r *http.Request) {
io.WriteString(w, "not json")
})
if _, err := ft.fetchAll(RequestOptions{
@@ -688,7 +688,7 @@ func TestFetchAllErrors(t *testing.T) {
var srvURL string
ft, srv := newTestFT(
t,
"warn",
false,
func(w http.ResponseWriter, r *http.Request) {
if r.URL.Query().Get("page") == "" {
io.WriteString(
@@ -730,7 +730,7 @@ func TestFetchAllRepeatedNextLink(t *testing.T) {
var srvURL string
ft, srv := newTestFT(
t,
"error",
false,
func(w http.ResponseWriter, r *http.Request) {
if hits.Add(1) > 10 {
t.Error("fetchAll did not stop following next links")