bwilliams
3a06132071
internal/eusupply replaces the old Node scraper (eu-supply.js) for uk.eu-supply.com. The site has no API, so it posts the public search form, which now needs the form's current defaults and reads the HTML with golang.org/x/net/html. Each tender not stored yet has its own page read too, for its description, contract type, CPV codes and buyer address. Tenders are stored with the source "EU Supply - UK" as eusupply-uk-<tender id>, with one release holding the tender as read. The site publishes no rate limit, so the client sets its own: one request at a time, with a random wait between sources.eu_supply.request_min_gap and request_max_gap (2s to 10s). The source has the same 7-day backfill as Find a Tender's. Most of its tenders are also on Find a Tender, so a new one is stored Hidden and Skipped when another source's opportunity released within 30 days has a title at least 90% alike, and its buyer or description matches too (80% of the shorter one's significant words).
1139 lines
31 KiB
Go
1139 lines
31 KiB
Go
package store_test
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"path/filepath"
|
|
"slices"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"git.avsdev.uk/Go/job-search/internal/store"
|
|
"git.avsdev.uk/Go/job-search/internal/types"
|
|
)
|
|
|
|
// Run with: go test -tags sqlite_fts5 ./...
|
|
|
|
func openStore(t *testing.T) *store.Store {
|
|
t.Helper()
|
|
path := filepath.Join(t.TempDir(), "test.db")
|
|
s, err := store.Open(context.Background(), path, store.WithAutoMigrate())
|
|
if err != nil {
|
|
t.Fatalf("Open: %v", err)
|
|
}
|
|
t.Cleanup(func() {
|
|
s.Close()
|
|
})
|
|
return s
|
|
}
|
|
|
|
func day(d int) time.Time {
|
|
return time.Date(2026, time.March, d, 9, 30, 0, 0, time.UTC)
|
|
}
|
|
|
|
func opportunity(
|
|
uuid, title string,
|
|
releaseDay int,
|
|
releaseIDs ...string,
|
|
) *types.Opportunity {
|
|
start, end := day(20), day(28)
|
|
t := &types.Opportunity{
|
|
UUID: uuid,
|
|
UUIDType: "ocid",
|
|
Title: title,
|
|
Status: "active",
|
|
Description: "A description",
|
|
BuyerName: "Example Council",
|
|
BuyerID: "GB-EXAMPLE",
|
|
MainCategory: "services",
|
|
ProcurementMethod: "open",
|
|
MultiLot: true,
|
|
SMESuitable: true,
|
|
StartDate: &start,
|
|
EndDate: &end,
|
|
GrossValue: 125000.50,
|
|
HasDocuments: true,
|
|
Source: "FindATender",
|
|
Classifications: []types.ClassificationCode{
|
|
{
|
|
IsCPV: true,
|
|
Code: "72000000",
|
|
Classification: "IT services",
|
|
},
|
|
{
|
|
IsCPV: true,
|
|
Code: "03000000",
|
|
Classification: "Agricultural products",
|
|
},
|
|
},
|
|
// No UUID: SaveOpportunity must fill it in.
|
|
Marks: types.OpportunityMarks{
|
|
Interested: true,
|
|
BidSubmitted: true,
|
|
},
|
|
}
|
|
|
|
for i, id := range releaseIDs {
|
|
t.Releases = append(t.Releases, types.OpportunityRelease{
|
|
ReleaseID: id,
|
|
ReleaseDate: day(releaseDay).Add(time.Duration(i) * time.Minute),
|
|
JSON: `{"id":"` + id + `"}`,
|
|
Tags: []types.OpportunityReleaseTag{
|
|
{
|
|
Tag: "opportunity",
|
|
},
|
|
{
|
|
Tag: "planning",
|
|
},
|
|
},
|
|
})
|
|
}
|
|
|
|
return t
|
|
}
|
|
|
|
func mustSave(
|
|
t *testing.T, s *store.Store, td *types.Opportunity,
|
|
) store.SaveResult {
|
|
t.Helper()
|
|
res, err := s.SaveOpportunity(context.Background(), td)
|
|
if err != nil {
|
|
t.Fatalf("SaveOpportunity(%s): %v", td.UUID, err)
|
|
}
|
|
return res
|
|
}
|
|
|
|
func mustGet(t *testing.T, s *store.Store, uuid string) *types.Opportunity {
|
|
t.Helper()
|
|
got, err := s.GetOpportunity(context.Background(), uuid)
|
|
if err != nil {
|
|
t.Fatalf("GetOpportunity(%s): %v", uuid, err)
|
|
}
|
|
return got
|
|
}
|
|
|
|
func uuids(ts []types.Opportunity) []string {
|
|
out := []string{}
|
|
for _, t := range ts {
|
|
out = append(out, t.UUID)
|
|
}
|
|
return out
|
|
}
|
|
|
|
func equal(a, b []string) bool {
|
|
return slices.Equal(a, b)
|
|
}
|
|
|
|
func TestSaveAndGetRoundTrip(t *testing.T) {
|
|
s := openStore(t)
|
|
in := opportunity("ocds-1", "Cloud migration services", 1, "r1", "r2")
|
|
res := mustSave(t, s, in)
|
|
if !res.OpportunityUpdated || res.NewReleases != 2 {
|
|
t.Fatalf("first save: got %+v", res)
|
|
}
|
|
|
|
got := mustGet(t, s, "ocds-1")
|
|
if got.Title != in.Title || got.BuyerName != in.BuyerName ||
|
|
got.BuyerID != in.BuyerID || got.MainCategory != in.MainCategory ||
|
|
got.ProcurementMethod != in.ProcurementMethod || got.UUIDType != "ocid" ||
|
|
got.Status != "active" || got.Description != in.Description ||
|
|
!got.MultiLot || !got.SMESuitable || got.VCSESuitable ||
|
|
!got.HasDocuments || got.GrossValue != in.GrossValue ||
|
|
got.Source != in.Source {
|
|
t.Errorf("opportunity fields differ:\n got %+v\nwant %+v", got, in)
|
|
}
|
|
if got.StartDate == nil || !got.StartDate.Equal(day(20)) ||
|
|
got.EndDate == nil || !got.EndDate.Equal(day(28)) {
|
|
t.Errorf("dates: got %v / %v", got.StartDate, got.EndDate)
|
|
}
|
|
if want := day(1).Add(time.Minute); !got.LatestReleaseDate.Equal(want) {
|
|
t.Errorf("LatestReleaseDate = %v, want %v", got.LatestReleaseDate, want)
|
|
}
|
|
|
|
if len(got.Classifications) != 2 ||
|
|
got.Classifications[0].Code != "03000000" ||
|
|
got.Classifications[1].Code != "72000000" ||
|
|
got.Classifications[0].ID == 0 {
|
|
t.Errorf(
|
|
"classifications: got %+v (leading zero must survive)",
|
|
got.Classifications,
|
|
)
|
|
}
|
|
|
|
wantMarks := types.OpportunityMarks{
|
|
UUID: "ocds-1",
|
|
Interested: true,
|
|
BidSubmitted: true,
|
|
}
|
|
if got.Marks != wantMarks {
|
|
t.Errorf("marks: got %+v, want %+v", got.Marks, wantMarks)
|
|
}
|
|
|
|
if len(got.Releases) != 2 {
|
|
t.Fatalf("releases: got %d", len(got.Releases))
|
|
}
|
|
r := got.Releases[0]
|
|
if r.ReleaseID != "r1" || r.UUID != "ocds-1" || r.JSON != `{"id":"r1"}` ||
|
|
!r.ReleaseDate.Equal(day(1)) {
|
|
t.Errorf("release: got %+v", r)
|
|
}
|
|
if len(r.Tags) != 2 || r.Tags[0].Tag != "opportunity" ||
|
|
r.Tags[1].Tag != "planning" || r.Tags[0].ReleaseID != "r1" ||
|
|
r.Tags[0].UUID != "ocds-1" {
|
|
t.Errorf("tags: got %+v", r.Tags)
|
|
}
|
|
}
|
|
|
|
func TestStartAndEndDateMayBeNil(t *testing.T) {
|
|
s := openStore(t)
|
|
in := opportunity("ocds-1", "No dates", 1, "r1")
|
|
in.StartDate, in.EndDate = nil, nil
|
|
mustSave(t, s, in)
|
|
got := mustGet(t, s, "ocds-1")
|
|
if got.StartDate != nil || got.EndDate != nil {
|
|
t.Errorf("got %v / %v, want nil", got.StartDate, got.EndDate)
|
|
}
|
|
}
|
|
|
|
func TestReleasesAreWriteOnce(t *testing.T) {
|
|
s := openStore(t)
|
|
mustSave(t, s, opportunity("ocds-1", "Title", 1, "r1"))
|
|
|
|
again := opportunity("ocds-1", "Title", 1, "r1")
|
|
again.Releases[0].JSON = `{"changed":true}`
|
|
again.Releases[0].Tags = []types.OpportunityReleaseTag{{
|
|
Tag: "award",
|
|
}}
|
|
if res := mustSave(t, s, again); res.NewReleases != 0 {
|
|
t.Errorf("re-save: NewReleases = %d, want 0", res.NewReleases)
|
|
}
|
|
|
|
got := mustGet(t, s, "ocds-1").Releases
|
|
if len(got) != 1 || got[0].JSON != `{"id":"r1"}` || len(got[0].Tags) != 2 {
|
|
t.Errorf("stored release changed: %+v", got)
|
|
}
|
|
}
|
|
|
|
func TestOlderReleaseDoesNotOverwriteOpportunity(t *testing.T) {
|
|
s := openStore(t)
|
|
mustSave(t, s, opportunity("ocds-1", "Newer title", 10, "r2"))
|
|
|
|
old := opportunity("ocds-1", "Older title", 5, "r1")
|
|
old.Classifications = old.Classifications[:1]
|
|
res := mustSave(t, s, old)
|
|
if res.OpportunityUpdated || res.NewReleases != 1 {
|
|
t.Fatalf("older save: got %+v", res)
|
|
}
|
|
|
|
got := mustGet(t, s, "ocds-1")
|
|
if got.Title != "Newer title" || len(got.Classifications) != 2 ||
|
|
!got.LatestReleaseDate.Equal(day(10)) {
|
|
t.Errorf("opportunity was overwritten by an older release: %+v", got)
|
|
}
|
|
if len(got.Releases) != 2 || got.Releases[0].ReleaseID != "r1" {
|
|
t.Errorf(
|
|
"older release should still be stored, oldest first: %+v", got.Releases,
|
|
)
|
|
}
|
|
}
|
|
|
|
func TestNewerReleaseUpdatesOpportunityAndSearchIndex(t *testing.T) {
|
|
s := openStore(t)
|
|
ctx := context.Background()
|
|
first := opportunity("ocds-1", "Cloud hosting", 1, "r1")
|
|
mustSave(t, s, first)
|
|
oldIT := first.Classifications[0].ID
|
|
|
|
newer := opportunity("ocds-1", "Website redesign", 2, "r2")
|
|
newer.Status = "complete"
|
|
newer.Classifications = []types.ClassificationCode{
|
|
{
|
|
IsCPV: true,
|
|
Code: "72000000",
|
|
Classification: "IT services (amended)",
|
|
},
|
|
}
|
|
if res := mustSave(t, s, newer); !res.OpportunityUpdated {
|
|
t.Fatalf("newer save not applied: %+v", res)
|
|
}
|
|
|
|
got := mustGet(t, s, "ocds-1")
|
|
if got.Title != "Website redesign" || got.Status != "complete" {
|
|
t.Errorf("opportunity not updated: %+v", got)
|
|
}
|
|
if len(got.Classifications) != 1 ||
|
|
got.Classifications[0].Classification != "IT services (amended)" {
|
|
t.Fatalf("classifications not replaced: %+v", got.Classifications)
|
|
}
|
|
if got.Classifications[0].ID == oldIT {
|
|
t.Errorf("changed description should create a new classification row")
|
|
}
|
|
|
|
for title, want := range map[string]int{"cloud": 0, "redesign": 1} {
|
|
ts, err := s.ListOpportunities(ctx, store.OpportunityFilter{
|
|
Title: title,
|
|
})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(ts) != want {
|
|
t.Errorf(
|
|
"search %q after title change: %d results, want %d",
|
|
title,
|
|
len(ts),
|
|
want,
|
|
)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestClassificationRowsAreShared(t *testing.T) {
|
|
s := openStore(t)
|
|
a := opportunity("ocds-a", "A", 1, "r1")
|
|
b := opportunity("ocds-b", "B", 1, "r1")
|
|
mustSave(t, s, a)
|
|
mustSave(t, s, b)
|
|
if a.Classifications[0].ID == 0 ||
|
|
a.Classifications[0].ID != b.Classifications[0].ID {
|
|
t.Errorf(
|
|
"identical classifications should share an ID: %d vs %d",
|
|
a.Classifications[0].ID,
|
|
b.Classifications[0].ID,
|
|
)
|
|
}
|
|
|
|
c := opportunity("ocds-c", "C", 1, "r1")
|
|
c.Classifications = []types.ClassificationCode{
|
|
{
|
|
IsCPV: false,
|
|
Code: "72000000",
|
|
Classification: "IT services",
|
|
},
|
|
}
|
|
mustSave(t, s, c)
|
|
if c.Classifications[0].ID == a.Classifications[0].ID {
|
|
t.Errorf("different is_cpv should not share an ID")
|
|
}
|
|
}
|
|
|
|
func TestSameReleaseIDInDifferentOpportunities(t *testing.T) {
|
|
s := openStore(t)
|
|
a := opportunity("ocds-a", "A", 1, "001")
|
|
b := opportunity("ocds-b", "B", 1, "001")
|
|
b.Releases[0].Tags = []types.OpportunityReleaseTag{{
|
|
Tag: "award",
|
|
}}
|
|
|
|
if res := mustSave(t, s, a); res.NewReleases != 1 {
|
|
t.Fatal(res)
|
|
}
|
|
if res := mustSave(t, s, b); res.NewReleases != 1 {
|
|
t.Fatalf("release id reused under another OCID must be stored: %+v", res)
|
|
}
|
|
|
|
got := mustGet(t, s, "ocds-b").Releases
|
|
if len(got) != 1 || len(got[0].Tags) != 1 || got[0].Tags[0].Tag != "award" {
|
|
t.Errorf("tags leaked between opportunities: %+v", got)
|
|
}
|
|
}
|
|
|
|
func TestListOpportunitiesFilters(t *testing.T) {
|
|
s := openStore(t)
|
|
ctx := context.Background()
|
|
|
|
cloud := opportunity("ocds-1", "Cloud migration services", 1, "r1")
|
|
roads := opportunity("ocds-2", "Road resurfacing works", 2, "r1")
|
|
roads.Status = "complete"
|
|
roads.Classifications = []types.ClassificationCode{
|
|
{
|
|
IsCPV: true,
|
|
Code: "45233142",
|
|
Classification: "Road-repair works",
|
|
},
|
|
}
|
|
hosting := opportunity("ocds-3", "Managed cloud hosting", 3, "r1")
|
|
for _, td := range []*types.Opportunity{cloud, roads, hosting} {
|
|
mustSave(t, s, td)
|
|
}
|
|
|
|
cases := []struct {
|
|
name string
|
|
f store.OpportunityFilter
|
|
want []string
|
|
}{
|
|
{
|
|
"all, newest first",
|
|
store.OpportunityFilter{},
|
|
[]string{"ocds-3", "ocds-2", "ocds-1"},
|
|
},
|
|
{"status", store.OpportunityFilter{
|
|
Status: "complete",
|
|
}, []string{"ocds-2"}},
|
|
{
|
|
"classification",
|
|
store.OpportunityFilter{
|
|
ClassificationCode: "72000000",
|
|
},
|
|
[]string{"ocds-3", "ocds-1"},
|
|
},
|
|
{"title word", store.OpportunityFilter{
|
|
Title: "cloud",
|
|
}, []string{"ocds-3", "ocds-1"}},
|
|
{"title prefixes", store.OpportunityFilter{
|
|
Title: "clo migr",
|
|
}, []string{"ocds-1"}},
|
|
{"title stemming", store.OpportunityFilter{
|
|
Title: "migrating",
|
|
}, []string{"ocds-1"}},
|
|
{"title case", store.OpportunityFilter{
|
|
Title: "ROAD",
|
|
}, []string{"ocds-2"}},
|
|
{
|
|
"title syntax is not FTS syntax",
|
|
store.OpportunityFilter{
|
|
Title: `cloud" OR title:road*`,
|
|
},
|
|
[]string{},
|
|
},
|
|
{
|
|
"title punctuation only",
|
|
store.OpportunityFilter{
|
|
Title: `"*:()`,
|
|
},
|
|
[]string{"ocds-3", "ocds-2", "ocds-1"},
|
|
},
|
|
{"released from", store.OpportunityFilter{
|
|
ReleasedFrom: day(2),
|
|
}, []string{"ocds-3", "ocds-2"}},
|
|
{"released before", store.OpportunityFilter{
|
|
ReleasedBefore: day(2),
|
|
}, []string{"ocds-1"}},
|
|
{"combined", store.OpportunityFilter{
|
|
Title: "cloud",
|
|
ReleasedFrom: day(2),
|
|
}, []string{"ocds-3"}},
|
|
{"limit", store.OpportunityFilter{
|
|
Limit: 2,
|
|
}, []string{"ocds-3", "ocds-2"}},
|
|
{"offset", store.OpportunityFilter{
|
|
Limit: 2,
|
|
Offset: 2,
|
|
}, []string{"ocds-1"}},
|
|
}
|
|
for _, tc := range cases {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
got, err := s.ListOpportunities(ctx, tc.f)
|
|
if err != nil {
|
|
t.Fatalf("ListOpportunities: %v", err)
|
|
}
|
|
if !equal(uuids(got), tc.want) {
|
|
t.Errorf("got %v, want %v", uuids(got), tc.want)
|
|
}
|
|
})
|
|
}
|
|
|
|
got, err := s.ListOpportunities(ctx, store.OpportunityFilter{
|
|
Status: "complete",
|
|
})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(got) != 1 || len(got[0].Classifications) != 1 ||
|
|
got[0].Releases != nil || got[0].Marks.UUID != "ocds-2" ||
|
|
!got[0].Marks.Interested {
|
|
t.Errorf(
|
|
"list should include classifications and marks and omit releases: %+v",
|
|
got,
|
|
)
|
|
}
|
|
}
|
|
|
|
func TestReleaseExistsAndListReleases(t *testing.T) {
|
|
s := openStore(t)
|
|
ctx := context.Background()
|
|
mustSave(t, s, opportunity("ocds-1", "T", 1, "r1", "r2"))
|
|
|
|
for _, tc := range []struct {
|
|
uuid, id string
|
|
want bool
|
|
}{{"ocds-1", "r1", true}, {"ocds-1", "r3", false}, {"ocds-2", "r1", false}} {
|
|
got, err := s.ReleaseExists(ctx, tc.uuid, tc.id)
|
|
if err != nil || got != tc.want {
|
|
t.Errorf(
|
|
"ReleaseExists(%s, %s) = %v, %v; want %v",
|
|
tc.uuid,
|
|
tc.id,
|
|
got,
|
|
err,
|
|
tc.want,
|
|
)
|
|
}
|
|
}
|
|
|
|
rs, err := s.ListReleases(ctx, "ocds-1")
|
|
if err != nil || len(rs) != 2 || rs[0].ReleaseID != "r1" ||
|
|
rs[1].ReleaseID != "r2" || len(rs[1].Tags) != 2 {
|
|
t.Errorf("ListReleases: %+v, %v", rs, err)
|
|
}
|
|
}
|
|
|
|
func TestGetOpportunityNotFound(t *testing.T) {
|
|
s := openStore(t)
|
|
_, err := s.GetOpportunity(context.Background(), "missing")
|
|
if !errors.Is(err, store.ErrNotFound) {
|
|
t.Errorf("err = %v, want ErrNotFound", err)
|
|
}
|
|
}
|
|
|
|
func TestSaveOpportunityValidation(t *testing.T) {
|
|
s := openStore(t)
|
|
ctx := context.Background()
|
|
for name, td := range map[string]*types.Opportunity{
|
|
"nil": nil,
|
|
"no uuid": opportunity("", "T", 1, "r1"),
|
|
"no release date": {UUID: "ocds-1", Title: "T", Status: "active"},
|
|
"no release id": opportunity("ocds-1", "T", 1, ""),
|
|
} {
|
|
if _, err := s.SaveOpportunity(ctx, td); err == nil {
|
|
t.Errorf("%s: expected an error", name)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestReopenKeepsDataAndSkipsAppliedMigrations(t *testing.T) {
|
|
path := filepath.Join(t.TempDir(), "test.db")
|
|
ctx := context.Background()
|
|
s, err := store.Open(ctx, path, store.WithAutoMigrate())
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
mustSave(t, s, opportunity("ocds-1", "Cloud migration", 1, "r1"))
|
|
s.Close()
|
|
|
|
// No WithAutoMigrate: an up-to-date database must pass the version check.
|
|
s, err = store.Open(ctx, path)
|
|
if err != nil {
|
|
t.Fatalf("reopen: %v", err)
|
|
}
|
|
defer s.Close()
|
|
got, err := s.ListOpportunities(ctx, store.OpportunityFilter{
|
|
Title: "cloud",
|
|
})
|
|
if err != nil || len(got) != 1 {
|
|
t.Errorf("after reopen: %v, %v", uuids(got), err)
|
|
}
|
|
}
|
|
|
|
func TestResaveLoadedOpportunity(t *testing.T) {
|
|
s := openStore(t)
|
|
mustSave(t, s, opportunity("ocds-1", "Original", 1, "r1"))
|
|
|
|
loaded := mustGet(t, s, "ocds-1") // classifications carry database IDs
|
|
ids := []int64{loaded.Classifications[0].ID, loaded.Classifications[1].ID}
|
|
loaded.Title = "Amended"
|
|
loaded.Releases = append(loaded.Releases, types.OpportunityRelease{
|
|
ReleaseID: "r2",
|
|
ReleaseDate: day(2),
|
|
JSON: `{}`,
|
|
Tags: []types.OpportunityReleaseTag{{
|
|
Tag: "opportunityAmendment",
|
|
}},
|
|
})
|
|
res := mustSave(t, s, loaded)
|
|
if !res.OpportunityUpdated || res.NewReleases != 1 {
|
|
t.Fatalf("resave: %+v", res)
|
|
}
|
|
|
|
got := mustGet(t, s, "ocds-1")
|
|
if got.Title != "Amended" || len(got.Releases) != 2 ||
|
|
!got.LatestReleaseDate.Equal(day(2)) {
|
|
t.Errorf("resave not applied: %+v", got)
|
|
}
|
|
if got.Classifications[0].ID != ids[0] ||
|
|
got.Classifications[1].ID != ids[1] {
|
|
t.Errorf(
|
|
"classification IDs changed on resave: %v -> %d, %d",
|
|
ids,
|
|
got.Classifications[0].ID,
|
|
got.Classifications[1].ID,
|
|
)
|
|
}
|
|
}
|
|
|
|
func mustSetMarks(t *testing.T, s *store.Store, m types.OpportunityMarks) {
|
|
t.Helper()
|
|
if err := s.SetMarks(context.Background(), m); err != nil {
|
|
t.Fatalf("SetMarks(%s): %v", m.UUID, err)
|
|
}
|
|
}
|
|
|
|
func TestSetMarks(t *testing.T) {
|
|
s := openStore(t)
|
|
mustSave(t, s, opportunity("ocds-1", "T", 1, "r1"))
|
|
|
|
for _, want := range []types.OpportunityMarks{
|
|
{UUID: "ocds-1", Hidden: true, Skipped: true},
|
|
{UUID: "ocds-1", Interested: true, BidSubmitted: true, Won: true},
|
|
{UUID: "ocds-1"},
|
|
} {
|
|
mustSetMarks(t, s, want)
|
|
if got := mustGet(t, s, "ocds-1").Marks; got != want {
|
|
t.Errorf("after SetMarks: got %+v, want %+v", got, want)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestSetMarksErrors(t *testing.T) {
|
|
s := openStore(t)
|
|
ctx := context.Background()
|
|
if err := s.SetMarks(
|
|
ctx,
|
|
types.OpportunityMarks{
|
|
UUID: "missing",
|
|
Hidden: true,
|
|
},
|
|
); !errors.Is(err, store.ErrNotFound) {
|
|
t.Errorf("unknown opportunity: err = %v, want ErrNotFound", err)
|
|
}
|
|
if err := s.SetMarks(ctx, types.OpportunityMarks{
|
|
Hidden: true,
|
|
}); err == nil {
|
|
t.Errorf("no UUID: expected an error")
|
|
}
|
|
}
|
|
|
|
// Fetched opportunities carry zero marks; saving them must not reset the
|
|
// marks the user has set.
|
|
func TestSaveKeepsExistingMarks(t *testing.T) {
|
|
s := openStore(t)
|
|
ctx := context.Background()
|
|
mustSave(t, s, opportunity("ocds-1", "T", 1, "r1"))
|
|
mustSave(t, s, opportunity("ocds-2", "T", 1, "r1"))
|
|
want := types.OpportunityMarks{
|
|
UUID: "ocds-1",
|
|
Hidden: true,
|
|
Lost: true,
|
|
}
|
|
mustSetMarks(t, s, want)
|
|
|
|
newer := opportunity("ocds-1", "Amended", 2, "r2")
|
|
newer.Marks = types.OpportunityMarks{}
|
|
if res := mustSave(t, s, newer); !res.OpportunityUpdated {
|
|
t.Fatalf("newer save not applied: %+v", res)
|
|
}
|
|
if got := mustGet(t, s, "ocds-1").Marks; got != want {
|
|
t.Errorf("SaveOpportunity changed marks: got %+v, want %+v", got, want)
|
|
}
|
|
|
|
newest := opportunity("ocds-1", "Amended again", 3, "r3")
|
|
newest.Marks = types.OpportunityMarks{}
|
|
if _, err := s.SaveOpportunities(
|
|
ctx, []*types.Opportunity{newest},
|
|
); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if got := mustGet(t, s, "ocds-1").Marks; got != want {
|
|
t.Errorf("SaveOpportunities changed marks: got %+v, want %+v", got, want)
|
|
}
|
|
}
|
|
|
|
func TestSaveOpportunitiesStoresMarks(t *testing.T) {
|
|
s := openStore(t)
|
|
a := opportunity("ocds-a", "A", 1, "r1")
|
|
b := opportunity("ocds-b", "B", 1, "r1")
|
|
b.Marks = types.OpportunityMarks{
|
|
Skipped: true,
|
|
}
|
|
|
|
if _, err := s.SaveOpportunities(
|
|
context.Background(), []*types.Opportunity{a, b},
|
|
); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
for uuid, want := range map[string]types.OpportunityMarks{
|
|
"ocds-a": {UUID: "ocds-a", Interested: true, BidSubmitted: true},
|
|
"ocds-b": {UUID: "ocds-b", Skipped: true},
|
|
} {
|
|
if got := mustGet(t, s, uuid).Marks; got != want {
|
|
t.Errorf("%s marks: got %+v, want %+v", uuid, got, want)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestListOpportunitiesMarkFilters(t *testing.T) {
|
|
s := openStore(t)
|
|
ctx := context.Background()
|
|
for i, uuid := range []string{"ocds-1", "ocds-2", "ocds-3", "ocds-4"} {
|
|
mustSave(t, s, opportunity(uuid, "Cloud hosting", i+1, "r1"))
|
|
}
|
|
// ocds-1 keeps the marks it was saved with: Interested, BidSubmitted.
|
|
mustSetMarks(t, s, types.OpportunityMarks{
|
|
UUID: "ocds-2",
|
|
Hidden: true,
|
|
})
|
|
mustSetMarks(t, s, types.OpportunityMarks{
|
|
UUID: "ocds-3",
|
|
Skipped: true,
|
|
Lost: true,
|
|
})
|
|
mustSetMarks(t, s, types.OpportunityMarks{
|
|
UUID: "ocds-4",
|
|
Interested: true,
|
|
BidSubmitted: true,
|
|
Won: true,
|
|
})
|
|
|
|
set, unset := store.MarkSet, store.MarkUnset
|
|
cases := []struct {
|
|
name string
|
|
f store.OpportunityFilter
|
|
want []string
|
|
}{
|
|
{
|
|
"any",
|
|
store.OpportunityFilter{},
|
|
[]string{"ocds-4", "ocds-3", "ocds-2", "ocds-1"},
|
|
},
|
|
{"hidden", store.OpportunityFilter{
|
|
Hidden: set,
|
|
}, []string{"ocds-2"}},
|
|
{"not hidden", store.OpportunityFilter{
|
|
Hidden: unset,
|
|
}, []string{"ocds-4", "ocds-3", "ocds-1"}},
|
|
{"skipped", store.OpportunityFilter{
|
|
Skipped: set,
|
|
}, []string{"ocds-3"}},
|
|
{
|
|
"not skipped",
|
|
store.OpportunityFilter{
|
|
Skipped: unset,
|
|
},
|
|
[]string{"ocds-4", "ocds-2", "ocds-1"},
|
|
},
|
|
{"interested", store.OpportunityFilter{
|
|
Interested: set,
|
|
}, []string{"ocds-4", "ocds-1"}},
|
|
{"not interested", store.OpportunityFilter{
|
|
Interested: unset,
|
|
}, []string{"ocds-3", "ocds-2"}},
|
|
{"bid submitted", store.OpportunityFilter{
|
|
BidSubmitted: set,
|
|
}, []string{"ocds-4", "ocds-1"}},
|
|
{
|
|
"bid not submitted",
|
|
store.OpportunityFilter{
|
|
BidSubmitted: unset,
|
|
},
|
|
[]string{"ocds-3", "ocds-2"},
|
|
},
|
|
{"won", store.OpportunityFilter{
|
|
Won: set,
|
|
}, []string{"ocds-4"}},
|
|
{"not won", store.OpportunityFilter{
|
|
Won: unset,
|
|
}, []string{"ocds-3", "ocds-2", "ocds-1"}},
|
|
{"lost", store.OpportunityFilter{
|
|
Lost: set,
|
|
}, []string{"ocds-3"}},
|
|
{"not lost", store.OpportunityFilter{
|
|
Lost: unset,
|
|
}, []string{"ocds-4", "ocds-2", "ocds-1"}},
|
|
{"two marks", store.OpportunityFilter{
|
|
BidSubmitted: set,
|
|
Won: unset,
|
|
}, []string{"ocds-1"}},
|
|
{"no match", store.OpportunityFilter{
|
|
Won: set,
|
|
Lost: set,
|
|
}, []string{}},
|
|
{
|
|
"with other filters",
|
|
store.OpportunityFilter{
|
|
Hidden: unset,
|
|
Title: "cloud",
|
|
ReleasedFrom: day(3),
|
|
},
|
|
[]string{"ocds-4", "ocds-3"},
|
|
},
|
|
}
|
|
for _, tc := range cases {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
got, err := s.ListOpportunities(ctx, tc.f)
|
|
if err != nil {
|
|
t.Fatalf("ListOpportunities: %v", err)
|
|
}
|
|
if !equal(uuids(got), tc.want) {
|
|
t.Errorf("got %v, want %v", uuids(got), tc.want)
|
|
}
|
|
})
|
|
}
|
|
|
|
got, err := s.ListOpportunities(ctx, store.OpportunityFilter{})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(got) != 4 || !got[0].Marks.Won || !got[1].Marks.Lost ||
|
|
!got[2].Marks.Hidden || got[3].Marks.Hidden || !got[3].Marks.Interested {
|
|
t.Errorf("list marks: %+v", got)
|
|
}
|
|
}
|
|
|
|
func TestWonAndLostAreExclusive(t *testing.T) {
|
|
s := openStore(t)
|
|
ctx := context.Background()
|
|
mustSave(t, s, opportunity("ocds-1", "T", 1, "r1"))
|
|
before := mustGet(t, s, "ocds-1").Marks
|
|
|
|
both := types.OpportunityMarks{
|
|
UUID: "ocds-1",
|
|
Won: true,
|
|
Lost: true,
|
|
}
|
|
if err := s.SetMarks(ctx, both); !errors.Is(err, types.ErrWonAndLost) {
|
|
t.Errorf("SetMarks(won and lost): err = %v, want ErrWonAndLost", err)
|
|
}
|
|
if got := mustGet(t, s, "ocds-1").Marks; got != before {
|
|
t.Errorf("rejected SetMarks changed marks: got %+v, want %+v", got, before)
|
|
}
|
|
|
|
in := opportunity("ocds-2", "T", 1, "r1")
|
|
in.Marks = types.OpportunityMarks{
|
|
Won: true,
|
|
Lost: true,
|
|
}
|
|
_, err := s.SaveOpportunity(ctx, in)
|
|
if !errors.Is(err, types.ErrWonAndLost) {
|
|
t.Errorf("SaveOpportunity(won and lost): err = %v, want ErrWonAndLost", err)
|
|
}
|
|
|
|
_, err = s.SaveOpportunities(ctx, []*types.Opportunity{in})
|
|
if !errors.Is(err, types.ErrWonAndLost) {
|
|
t.Errorf(
|
|
"SaveOpportunities(won and lost): err = %v, want ErrWonAndLost", err,
|
|
)
|
|
}
|
|
|
|
_, err = s.GetOpportunity(ctx, "ocds-2")
|
|
if !errors.Is(err, store.ErrNotFound) {
|
|
t.Errorf("rejected opportunity was stored: err = %v", err)
|
|
}
|
|
|
|
// Either one alone is fine, and switching between them works.
|
|
for _, m := range []types.OpportunityMarks{
|
|
{UUID: "ocds-1", Won: true},
|
|
{UUID: "ocds-1", Lost: true},
|
|
} {
|
|
mustSetMarks(t, s, m)
|
|
if got := mustGet(t, s, "ocds-1").Marks; got != m {
|
|
t.Errorf("got %+v, want %+v", got, m)
|
|
}
|
|
}
|
|
}
|
|
|
|
// LatestReleaseDate is the newest release of one source's opportunities only,
|
|
// and the zero time for a source with none stored.
|
|
func TestLatestReleaseDate(t *testing.T) {
|
|
s := openStore(t)
|
|
ctx := context.Background()
|
|
|
|
latest, err := s.LatestReleaseDate(ctx, "FindATender")
|
|
if err != nil || !latest.IsZero() {
|
|
t.Fatalf("empty store: %v, %v; want the zero time", latest, err)
|
|
}
|
|
|
|
mustSave(t, s, opportunity("ocds-1", "Older", 1, "r1"))
|
|
mustSave(t, s, opportunity("ocds-2", "Newer", 5, "r1"))
|
|
other := opportunity("ocds-3", "Another source's", 9, "r1")
|
|
other.Source = "FTSMock"
|
|
mustSave(t, s, other)
|
|
|
|
for source, uuid := range map[string]string{
|
|
"FindATender": "ocds-2",
|
|
"FTSMock": "ocds-3",
|
|
} {
|
|
want := mustGet(t, s, uuid).LatestReleaseDate
|
|
latest, err := s.LatestReleaseDate(ctx, source)
|
|
if err != nil || !latest.Equal(want) {
|
|
t.Errorf("%s: %v, %v; want %v", source, latest, err, want)
|
|
}
|
|
}
|
|
|
|
latest, err = s.LatestReleaseDate(ctx, "Nowhere")
|
|
if err != nil || !latest.IsZero() {
|
|
t.Errorf("unknown source: %v, %v; want the zero time", latest, err)
|
|
}
|
|
}
|
|
|
|
// ListOtherSourcesTitles lists only other sources' opportunities, with
|
|
// their latest release in [from, before).
|
|
func TestListOtherSourcesTitles(t *testing.T) {
|
|
s := openStore(t)
|
|
ctx := context.Background()
|
|
|
|
mustSave(t, s, opportunity("ocds-1", "Too early", 1, "r1"))
|
|
mustSave(t, s, opportunity("ocds-2", "In the window", 3, "r1"))
|
|
mustSave(t, s, opportunity("ocds-3", "At the end", 5, "r1"))
|
|
own := opportunity("eusupply-uk-1", "Its own source", 3, "r1")
|
|
own.Source = "EU Supply - UK"
|
|
mustSave(t, s, own)
|
|
|
|
titles, err := s.ListOtherSourcesTitles(
|
|
ctx, "EU Supply - UK", day(2), day(5),
|
|
)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(titles) != 1 || titles[0].UUID != "ocds-2" ||
|
|
titles[0].Title != "In the window" ||
|
|
!titles[0].LatestReleaseDate.Equal(day(3)) {
|
|
t.Errorf("titles = %+v; want only ocds-2", titles)
|
|
}
|
|
}
|
|
|
|
// NewOpportunity is set only for an opportunity that wasn't already stored,
|
|
// by both SaveOpportunity and SaveOpportunities.
|
|
func TestSaveResultNewOpportunity(t *testing.T) {
|
|
s := openStore(t)
|
|
ctx := context.Background()
|
|
|
|
res := mustSave(t, s, opportunity("ocds-1", "First", 1, "r1"))
|
|
if !res.NewOpportunity {
|
|
t.Errorf("first save: NewOpportunity = false")
|
|
}
|
|
res = mustSave(t, s, opportunity("ocds-1", "First", 2, "r2"))
|
|
if res.NewOpportunity {
|
|
t.Errorf("second save: NewOpportunity = true")
|
|
}
|
|
|
|
total, err := s.SaveOpportunities(ctx, []*types.Opportunity{
|
|
opportunity("ocds-1", "First", 3, "r3"),
|
|
opportunity("ocds-2", "Second", 3, "r1"),
|
|
})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
results := total.OpportunityResults
|
|
if results[0].NewOpportunity || !results[1].NewOpportunity {
|
|
t.Errorf(
|
|
"SaveOpportunities results = %+v; want only the second new",
|
|
results,
|
|
)
|
|
}
|
|
}
|
|
|
|
// ListClassifications lists the codes in use, once each and sorted, and not
|
|
// a code no opportunity has any more.
|
|
func TestListClassifications(t *testing.T) {
|
|
s := openStore(t)
|
|
ctx := context.Background()
|
|
|
|
got, err := s.ListClassifications(ctx)
|
|
if err != nil || got == nil || len(got) != 0 {
|
|
t.Fatalf("empty store: %v, %v; want an empty, non-nil list", got, err)
|
|
}
|
|
|
|
mustSave(t, s, opportunity("ocds-1", "First", 1, "r1"))
|
|
mustSave(t, s, opportunity("ocds-2", "Second", 1, "r1"))
|
|
codes := func() []string {
|
|
t.Helper()
|
|
got, err := s.ListClassifications(ctx)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var listed []string
|
|
for _, classification := range got {
|
|
listed = append(listed, classification.Code)
|
|
}
|
|
return listed
|
|
}
|
|
if got := strings.Join(codes(), ","); got != "03000000,72000000" {
|
|
t.Fatalf("codes = %s; want 03000000,72000000", got)
|
|
}
|
|
|
|
// Newer releases replace both opportunities' classifications with one
|
|
// code; the other is still stored, but no longer listed.
|
|
for _, uuid := range []string{"ocds-1", "ocds-2"} {
|
|
newer := opportunity(uuid, "Newer", 2, "r2")
|
|
newer.Classifications = newer.Classifications[1:]
|
|
mustSave(t, s, newer)
|
|
}
|
|
if got := strings.Join(codes(), ","); got != "03000000" {
|
|
t.Errorf("after the change: codes = %s; want 03000000", got)
|
|
}
|
|
}
|
|
|
|
// SaveOpportunities reports the classification codes a save brings into use
|
|
// and those it takes out of use, and none when the set doesn't change.
|
|
func TestSaveOpportunitiesClassificationChanges(t *testing.T) {
|
|
s := openStore(t)
|
|
ctx := context.Background()
|
|
save := func(opportunity *types.Opportunity) store.SaveMultipleResult {
|
|
t.Helper()
|
|
total, err := s.SaveOpportunities(ctx, []*types.Opportunity{opportunity})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return total
|
|
}
|
|
codesOf := func(classifications []types.ClassificationCode) string {
|
|
var codes []string
|
|
for _, classification := range classifications {
|
|
codes = append(codes, classification.Code)
|
|
}
|
|
return strings.Join(codes, ",")
|
|
}
|
|
|
|
total := save(opportunity("ocds-1", "First", 1, "r1"))
|
|
if codesOf(total.AddedClassifications) != "03000000,72000000" ||
|
|
total.RemovedClassificationIDs != nil {
|
|
t.Fatalf(
|
|
"first save: added %v, removed %v; want both codes added",
|
|
total.AddedClassifications, total.RemovedClassificationIDs,
|
|
)
|
|
}
|
|
removedID := total.AddedClassifications[1].ID // 72000000
|
|
|
|
total = save(opportunity("ocds-2", "Second", 1, "r1"))
|
|
if total.AddedClassifications != nil ||
|
|
total.RemovedClassificationIDs != nil {
|
|
t.Errorf(
|
|
"codes already in use: added %v, removed %v; want none",
|
|
total.AddedClassifications, total.RemovedClassificationIDs,
|
|
)
|
|
}
|
|
|
|
// ocds-1 drops 72000000, which ocds-2 still has: still in use.
|
|
newer := opportunity("ocds-1", "Newer", 2, "r2")
|
|
newer.Classifications = newer.Classifications[1:]
|
|
total = save(newer)
|
|
if total.AddedClassifications != nil ||
|
|
total.RemovedClassificationIDs != nil {
|
|
t.Errorf(
|
|
"code still in use: added %v, removed %v; want none",
|
|
total.AddedClassifications, total.RemovedClassificationIDs,
|
|
)
|
|
}
|
|
|
|
// ocds-2 drops it too: it goes out of use.
|
|
newer = opportunity("ocds-2", "Newer", 2, "r2")
|
|
newer.Classifications = newer.Classifications[1:]
|
|
total = save(newer)
|
|
if total.AddedClassifications != nil ||
|
|
len(total.RemovedClassificationIDs) != 1 ||
|
|
total.RemovedClassificationIDs[0] != removedID {
|
|
t.Errorf(
|
|
"code out of use: added %v, removed %v; want removed [%d]",
|
|
total.AddedClassifications, total.RemovedClassificationIDs, removedID,
|
|
)
|
|
}
|
|
}
|
|
|
|
// A save records the opportunities it adds or gives new releases in a change
|
|
// batch, as does a change of marks, and ListChangedOpportunities returns
|
|
// those in the batches after a given one.
|
|
func TestChangeBatches(t *testing.T) {
|
|
s := openStore(t)
|
|
ctx := context.Background()
|
|
save := func(opportunities ...*types.Opportunity) string {
|
|
t.Helper()
|
|
total, err := s.SaveOpportunities(ctx, opportunities)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return total.ChangeBatch
|
|
}
|
|
changed := func(after string) ([]types.Opportunity, string) {
|
|
t.Helper()
|
|
opportunities, latest, err := s.ListChangedOpportunities(
|
|
ctx, after, time.Time{},
|
|
)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return opportunities, latest
|
|
}
|
|
latest := func() string {
|
|
t.Helper()
|
|
hash, err := s.LatestChangeBatch(ctx)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return hash
|
|
}
|
|
|
|
if hash := latest(); hash != "" {
|
|
t.Fatalf("empty store: latest batch %q, want none", hash)
|
|
}
|
|
|
|
first := save(
|
|
opportunity("ocds-1", "One", 1, "r1"),
|
|
opportunity("ocds-2", "Two", 1, "r1"),
|
|
)
|
|
if len(first) != 64 || strings.Trim(first, "0123456789abcdef") != "" {
|
|
t.Fatalf("batch %q is not a hex SHA-256 hash", first)
|
|
}
|
|
if hash := latest(); hash != first {
|
|
t.Errorf("latest batch %q, want %q", hash, first)
|
|
}
|
|
|
|
// Nothing new: no batch.
|
|
if hash := save(opportunity("ocds-1", "One", 1, "r1")); hash != "" {
|
|
t.Errorf("save with nothing new recorded batch %q", hash)
|
|
}
|
|
|
|
second := save(
|
|
opportunity("ocds-2", "Two", 2, "r1", "r2"),
|
|
opportunity("ocds-3", "Three", 3, "r1"),
|
|
)
|
|
if second == "" || second == first {
|
|
t.Fatalf("second batch %q, first %q", second, first)
|
|
}
|
|
|
|
got, hash := changed(first)
|
|
if want := []string{"ocds-3", "ocds-2"}; !equal(uuids(got), want) {
|
|
t.Errorf("changed after first: %v, want %v", uuids(got), want)
|
|
}
|
|
if hash != second {
|
|
t.Errorf("changed after first: latest %q, want %q", hash, second)
|
|
}
|
|
if len(got) == 2 && len(got[1].Classifications) != 2 {
|
|
t.Errorf("changed opportunity without classifications: %+v", got[1])
|
|
}
|
|
|
|
// releasedFrom leaves out ocds-2 (released day 2), not the batch.
|
|
got, hash, err := s.ListChangedOpportunities(ctx, first, day(3))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if !equal(uuids(got), []string{"ocds-3"}) || hash != second {
|
|
t.Errorf(
|
|
"changed from day 3: %v, %q; want [ocds-3], %q",
|
|
uuids(got), hash, second,
|
|
)
|
|
}
|
|
|
|
mustSetMarks(t, s, types.OpportunityMarks{
|
|
UUID: "ocds-1",
|
|
Skipped: true,
|
|
})
|
|
got, hash = changed(second)
|
|
if !equal(uuids(got), []string{"ocds-1"}) || !got[0].Marks.Skipped {
|
|
t.Errorf("changed after marks: %+v, want ocds-1 skipped", got)
|
|
}
|
|
if hash == second || hash != latest() {
|
|
t.Errorf("changed after marks: latest %q, want a newer batch", hash)
|
|
}
|
|
|
|
got, again := changed(hash)
|
|
if got == nil || len(got) != 0 || again != hash {
|
|
t.Errorf("changed after latest: %v, %q; want [], %q", got, again, hash)
|
|
}
|
|
|
|
_, _, err = s.ListChangedOpportunities(ctx, "unknown", time.Time{})
|
|
if !errors.Is(err, store.ErrUnknownBatch) {
|
|
t.Errorf("unknown batch: err %v, want ErrUnknownBatch", err)
|
|
}
|
|
}
|