package mock import ( "bytes" "encoding/json" "math/rand" "net/http/httptest" "os" "path/filepath" "strings" "testing" "time" "git.avsdev.uk/Go/job-search/internal/mock/findatender/ocds" ) func TestStorePersistsAndGeneratorProgresses(t *testing.T) { cat := loadTestCatalog(t) path := filepath.Join(t.TempDir(), "releases.jsonl") store, err := OpenStore(path) if err != nil { t.Fatal(err) } gen := NewGenerator(cat, store, 42) now := time.Date(2026, 10, 8, 12, 0, 0, 0, time.UTC) n, err := gen.CatchUp(now, 48*time.Hour, 3*time.Minute, 15*time.Minute) if err != nil || n < 48*60/15 { t.Fatalf("catch-up generated %d (err %v)", n, err) } tags := map[string]int{} ocids := map[string]int{} all, _, _ := store.Page(time.Time{}, now, 0, 100000) for _, r := range all { tags[r.Tag[0]]++ ocids[r.OCID]++ if r.Tender == nil || r.Tender.Classification == nil { t.Fatalf("release %s has no tender.classification", r.ID) } } if tags["award"] == 0 || tags["tender"] == 0 { t.Fatalf("expected tenders and awards, got %v", tags) } if len(ocids) == len(all) { t.Fatal("expected some procedures with more than one release") } store.Close() // Reopen: everything is still there, and catch-up only fills the new gap. store2, err := OpenStore(path) if err != nil { t.Fatal(err) } defer store2.Close() if store2.Len() != len(all) { t.Fatalf("after reopen: %d releases, want %d", store2.Len(), len(all)) } gen2 := NewGenerator(cat, store2, 7) m, _ := gen2.CatchUp( now.Add(time.Hour), 48*time.Hour, 3*time.Minute, 15*time.Minute, ) if m < 4 || m > 20 { t.Fatalf("one hour gap produced %d releases", m) } if last, _ := store2.Last(); last.After(now.Add(time.Hour)) { t.Fatal("release dated in the future") } // The HTTP handler pages through the store without gaps or repeats. srv := &server{ store: store2, baseURL: "", location: time.UTC, } seen := map[string]bool{} next := BasePath + "?updatedFrom=2026-10-07T00:00:00&limit=7" for pages := 0; next != ""; pages++ { rec := httptest.NewRecorder() srv.handle(rec, httptest.NewRequest("GET", next, nil)) // Strict decode: the output must contain only fields the ocds types define. var pkg ocds.ReleasePackage dec := json.NewDecoder(bytes.NewReader(rec.Body.Bytes())) dec.DisallowUnknownFields() if err := dec.Decode(&pkg); err != nil { t.Fatalf("response doesn't match ocds types: %v", err) } for _, r := range pkg.Releases { if seen[r.ID] { t.Fatalf("release %s returned twice", r.ID) } seen[r.ID] = true } next = "" if pkg.Links != nil { next = pkg.Links.Next } } from := time.Date(2026, 10, 7, 0, 0, 0, 0, time.UTC) want, _, _ := store2.Page(from, time.Now(), 0, 100000) if len(seen) != len(want) { t.Fatalf("paged %d releases, want %d", len(seen), len(want)) } } func writeLines(t *testing.T, path string, lines ...string) { t.Helper() if err := os.WriteFile( path, []byte(strings.Join(lines, "\n")), 0o600, ); err != nil { t.Fatal(err) } } func releaseLine(t *testing.T, ocid, id, tag string, at time.Time) string { t.Helper() b, err := json.Marshal(ocds.Release{ OCID: ocid, ID: id, Date: at, Tag: []string{tag}, }) if err != nil { t.Fatal(err) } return string(b) } // Unreadable lines are skipped, blank lines ignored, a hand-edited file is put // back in date order, and the index (open procedures, notice numbers) is // rebuilt. func TestOpenStoreRecoversFromMessyFile(t *testing.T) { path := filepath.Join(t.TempDir(), "r.jsonl") t0 := time.Date(2026, 10, 8, 9, 0, 0, 0, time.UTC) writeLines( t, path, releaseLine(t, "ocds-b", "000007-2026", "tender", t0.Add(2*time.Hour)), "{not json", "", releaseLine(t, "ocds-a", "000003-2026", "tender", t0), releaseLine(t, "ocds-a", "000005-2026", "award", t0.Add(time.Hour)), "", ) store, err := OpenStore(path) if err != nil { t.Fatal(err) } defer store.Close() all, _, _ := store.Page(time.Time{}, t0.Add(24*time.Hour), 0, 100) var ids []string for _, r := range all { ids = append(ids, r.ID) } want := "000003-2026,000005-2026,000007-2026" if got := strings.Join(ids, ","); got != want { t.Fatalf("releases %s, want %s", got, want) } if got := store.NextNoticeID(2026); got != "000008-2026" { t.Errorf("NextNoticeID = %s, want 000008-2026", got) } if got := store.NextNoticeID(2027); got != "000001-2027" { t.Errorf("NextNoticeID(2027) = %s", got) } // ocds-a was awarded, so only ocds-b is still open. //nolint:gosec // G404: seeded math/rand keeps the test deterministic r, ok := store.RandomOpen(rand.New(rand.NewSource(1)), t0) if !ok || r.OCID != "ocds-b" { t.Errorf("RandomOpen = %s, %v; want ocds-b", r.OCID, ok) } } // A record cut off mid-write is skipped, and the next record starts on a fresh // line so it is still readable after a restart. func TestOpenStoreAfterTruncatedWrite(t *testing.T) { path := filepath.Join(t.TempDir(), "r.jsonl") t0 := time.Date(2026, 10, 8, 9, 0, 0, 0, time.UTC) full := releaseLine(t, "ocds-a", "000001-2026", "tender", t0) cut := releaseLine(t, "ocds-b", "000002-2026", "tender", t0.Add(time.Minute)) writeLines(t, path, full, cut[:len(cut)/2]) // no trailing newline store, err := OpenStore(path) if err != nil { t.Fatal(err) } if store.Len() != 1 { t.Fatalf("Len = %d, want 1", store.Len()) } if err := store.Append( ocds.Release{ OCID: "ocds-c", ID: "000002-2026", Date: t0.Add(2 * time.Minute), Tag: []string{"tender"}, }, ); err != nil { t.Fatal(err) } store.Close() store2, err := OpenStore(path) if err != nil { t.Fatal(err) } defer store2.Close() if store2.Len() != 2 || !store2.HasOCID("ocds-c") { t.Fatalf( "after reopen: Len = %d, has ocds-c = %v", store2.Len(), store2.HasOCID("ocds-c"), ) } } func TestOpenStoreErrors(t *testing.T) { dir := t.TempDir() // The parent "directory" is a regular file. file := filepath.Join(dir, "file") if err := os.WriteFile(file, nil, 0o600); err != nil { t.Fatal(err) } if _, err := OpenStore(filepath.Join(file, "r.jsonl")); err == nil { t.Error("parent is a file: expected error") } // The store path is a directory, which opens but can't be read. if _, err := OpenStore(dir); err == nil { t.Error("path is a directory: expected error") } // A missing parent directory is created. store, err := OpenStore(filepath.Join(dir, "a", "b", "r.jsonl")) if err != nil { t.Fatalf("nested path: %v", err) } store.Close() } // A release dated before the newest one is moved up to keep the order valid. func TestAppendKeepsDateOrder(t *testing.T) { store, err := OpenStore(filepath.Join(t.TempDir(), "r.jsonl")) if err != nil { t.Fatal(err) } defer store.Close() t0 := time.Date(2026, 10, 8, 9, 0, 0, 0, time.UTC) store.Append(ocds.Release{ OCID: "a", ID: "000001-2026", Date: t0, Tag: []string{"tender"}, }) store.Append( ocds.Release{ OCID: "b", ID: "000002-2026", Date: t0.Add(-time.Hour), Tag: []string{"tender"}, }, ) if last, _ := store.Last(); !last.Equal(t0) { t.Errorf("Last = %s, want %s", last, t0) } // Appending to a closed store fails and leaves the store unchanged. store.Close() if err := store.Append(ocds.Release{ OCID: "c", Date: t0, Tag: []string{"tender"}, }); err == nil { t.Error("append after close: expected error") } if store.Len() != 2 || store.HasOCID("c") { t.Errorf("failed append changed the store: Len = %d", store.Len()) } } // Procedures idle since before the cut-off are dropped from the open set. func TestRandomOpenDropsStaleProcedures(t *testing.T) { store, err := OpenStore(filepath.Join(t.TempDir(), "r.jsonl")) if err != nil { t.Fatal(err) } defer store.Close() t0 := time.Date(2026, 10, 8, 9, 0, 0, 0, time.UTC) store.Append(ocds.Release{ OCID: "old", ID: "000001-2026", Date: t0, Tag: []string{"tender"}, }) store.Append( ocds.Release{ OCID: "new", ID: "000002-2026", Date: t0.Add(time.Hour), Tag: []string{"tender"}, }, ) //nolint:gosec // G404: seeded math/rand keeps the test deterministic rnd := rand.New(rand.NewSource(1)) for i := 0; i < 10; i++ { if r, ok := store.RandomOpen(rnd, t0.Add(time.Minute)); !ok || r.OCID != "new" { t.Fatalf("RandomOpen = %s, %v; want new", r.OCID, ok) } } if _, ok := store.RandomOpen(rnd, t0.Add(2*time.Hour)); ok { t.Error("everything is stale: expected none") } }