Made each source start from its newest stored release, with --since as an override
Without --since, fetcher service and job-search serve start each source from the newest release stored for it (the new store.LatestReleaseDate, by Source), so a restart carries on where it left off instead of fetching the last day again; with nothing stored, the client's default still applies. --since overrides it for every source. The Find a Tender source's name is now a constant, as the Source its opportunities are stored with. The README's guide to adding a source is brought up to date with the current worker, start and config type names.
This commit is contained in:
1 parent
3c4fde8dfd
commit
284a589627
10 files changed
+256
-22
No files matched your search
@@ -552,3 +552,110 @@ func TestSave(t *testing.T) {
|
||||
t.Errorf("save to a closed store = %v; want a saving error", err)
|
||||
}
|
||||
}
|
||||
|
||||
// updatedFromServer serves emptyPackage and sends the updatedFrom of each
|
||||
// request to the returned channel.
|
||||
func updatedFromServer(t *testing.T) (*httptest.Server, chan string) {
|
||||
t.Helper()
|
||||
updatedFrom := make(chan string, 10)
|
||||
srv := httptest.NewServer(
|
||||
http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
select {
|
||||
case updatedFrom <- r.URL.Query().Get("updatedFrom"):
|
||||
default:
|
||||
}
|
||||
io.WriteString(w, emptyPackage)
|
||||
}),
|
||||
)
|
||||
t.Cleanup(srv.Close)
|
||||
return srv, updatedFrom
|
||||
}
|
||||
|
||||
// Without since, each source starts from its own newest stored release, or
|
||||
// with nothing stored from the client's default (the last day); since
|
||||
// overrides both.
|
||||
func TestRunStartsFromNewestStoredRelease(t *testing.T) {
|
||||
opportunityStore := openTestStore(t)
|
||||
srv, _ := ftsServer(t, http.StatusOK, onePackage)
|
||||
stored, err := fetchers["FindATender"](
|
||||
context.Background(), srv.URL, time.Now().Add(-time.Hour),
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, err = opportunityStore.SaveOpportunities(context.Background(), stored)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
override := time.Date(2026, 9, 1, 0, 0, 0, 0, time.UTC)
|
||||
tests := []struct {
|
||||
name string
|
||||
since time.Time
|
||||
wantFAT string
|
||||
wantMock string
|
||||
}{
|
||||
// "" is the client's default: about a day ago.
|
||||
{"stored", time.Time{}, "2026-10-01T12:00:00", ""},
|
||||
{"override", override, "2026-09-01T00:00:00", "2026-09-01T00:00:00"},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
fat, fatFrom := updatedFromServer(t)
|
||||
mock, mockFrom := updatedFromServer(t)
|
||||
sources := &config.JSCSources{}
|
||||
sources.FindATender.Enabled = true
|
||||
sources.FindATender.Interval = time.Hour
|
||||
sources.FindATender.APIRoot = fat.URL
|
||||
sources.FTSMock.Enabled = true
|
||||
sources.FTSMock.Interval = time.Hour
|
||||
sources.FTSMock.APIRoot = mock.URL
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
done := make(chan error, 1)
|
||||
go func() {
|
||||
done <- New(sources, WithStore(opportunityStore)).Run(ctx, tt.since)
|
||||
}()
|
||||
|
||||
for source, got := range map[string]chan string{
|
||||
"FindATender": fatFrom,
|
||||
"FTSMock": mockFrom,
|
||||
} {
|
||||
want := tt.wantFAT
|
||||
if source == "FTSMock" {
|
||||
want = tt.wantMock
|
||||
}
|
||||
select {
|
||||
case from := <-got:
|
||||
if want == "" {
|
||||
checkAboutADayAgo(t, from)
|
||||
} else if from != want {
|
||||
t.Errorf(
|
||||
"%s: %s updatedFrom = %s; want %s",
|
||||
tt.name, source, from, want,
|
||||
)
|
||||
}
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Errorf("%s: %s was not polled", tt.name, source)
|
||||
}
|
||||
}
|
||||
cancel()
|
||||
<-done
|
||||
}
|
||||
}
|
||||
|
||||
// checkAboutADayAgo fails the test unless updatedFrom, in local time as the
|
||||
// client sends it, is about a day ago: within an hour, as a calendar day
|
||||
// across a clock change isn't 24 hours.
|
||||
func checkAboutADayAgo(t *testing.T, updatedFrom string) {
|
||||
t.Helper()
|
||||
from, err := time.ParseInLocation(
|
||||
"2006-01-02T15:04:05", updatedFrom, time.Local,
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
gap := time.Since(from) - 24*time.Hour
|
||||
if gap < -time.Hour || gap > time.Hour {
|
||||
t.Errorf("updatedFrom = %s; want about a day ago", updatedFrom)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user