Made each source start from its newest stored release, with --since as an override

Without --since, fetcher service and job-search serve start each source
from the newest release stored for it (the new store.LatestReleaseDate,
by Source), so a restart carries on where it left off instead of fetching
the last day again; with nothing stored, the client's default still
applies. --since overrides it for every source. The Find a Tender source's
name is now a constant, as the Source its opportunities are stored with.

The README's guide to adding a source is brought up to date with the
current worker, start and config type names.
This commit is contained in:
bwilliams committed 2026-10-16 11:37:09 +01:00
1 parent 3c4fde8dfd
commit 284a589627
10 files changed
+256 -22

No files matched your search

+107
View File
@@ -552,3 +552,110 @@ func TestSave(t *testing.T) {
t.Errorf("save to a closed store = %v; want a saving error", err)
}
}
// updatedFromServer serves emptyPackage and sends the updatedFrom of each
// request to the returned channel.
func updatedFromServer(t *testing.T) (*httptest.Server, chan string) {
t.Helper()
updatedFrom := make(chan string, 10)
srv := httptest.NewServer(
http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
select {
case updatedFrom <- r.URL.Query().Get("updatedFrom"):
default:
}
io.WriteString(w, emptyPackage)
}),
)
t.Cleanup(srv.Close)
return srv, updatedFrom
}
// Without since, each source starts from its own newest stored release, or
// with nothing stored from the client's default (the last day); since
// overrides both.
func TestRunStartsFromNewestStoredRelease(t *testing.T) {
opportunityStore := openTestStore(t)
srv, _ := ftsServer(t, http.StatusOK, onePackage)
stored, err := fetchers["FindATender"](
context.Background(), srv.URL, time.Now().Add(-time.Hour),
)
if err != nil {
t.Fatal(err)
}
_, err = opportunityStore.SaveOpportunities(context.Background(), stored)
if err != nil {
t.Fatal(err)
}
override := time.Date(2026, 9, 1, 0, 0, 0, 0, time.UTC)
tests := []struct {
name string
since time.Time
wantFAT string
wantMock string
}{
// "" is the client's default: about a day ago.
{"stored", time.Time{}, "2026-10-01T12:00:00", ""},
{"override", override, "2026-09-01T00:00:00", "2026-09-01T00:00:00"},
}
for _, tt := range tests {
fat, fatFrom := updatedFromServer(t)
mock, mockFrom := updatedFromServer(t)
sources := &config.JSCSources{}
sources.FindATender.Enabled = true
sources.FindATender.Interval = time.Hour
sources.FindATender.APIRoot = fat.URL
sources.FTSMock.Enabled = true
sources.FTSMock.Interval = time.Hour
sources.FTSMock.APIRoot = mock.URL
ctx, cancel := context.WithCancel(context.Background())
done := make(chan error, 1)
go func() {
done <- New(sources, WithStore(opportunityStore)).Run(ctx, tt.since)
}()
for source, got := range map[string]chan string{
"FindATender": fatFrom,
"FTSMock": mockFrom,
} {
want := tt.wantFAT
if source == "FTSMock" {
want = tt.wantMock
}
select {
case from := <-got:
if want == "" {
checkAboutADayAgo(t, from)
} else if from != want {
t.Errorf(
"%s: %s updatedFrom = %s; want %s",
tt.name, source, from, want,
)
}
case <-time.After(5 * time.Second):
t.Errorf("%s: %s was not polled", tt.name, source)
}
}
cancel()
<-done
}
}
// checkAboutADayAgo fails the test unless updatedFrom, in local time as the
// client sends it, is about a day ago: within an hour, as a calendar day
// across a clock change isn't 24 hours.
func checkAboutADayAgo(t *testing.T, updatedFrom string) {
t.Helper()
from, err := time.ParseInLocation(
"2006-01-02T15:04:05", updatedFrom, time.Local,
)
if err != nil {
t.Fatal(err)
}
gap := time.Since(from) - 24*time.Hour
if gap < -time.Hour || gap > time.Hour {
t.Errorf("updatedFrom = %s; want about a day ago", updatedFrom)
}
}