package fetcher import ( "math/rand" "testing" "git.avsdev.uk/Go/job-search/internal/types" ) func TestNormaliseTitle(t *testing.T) { tests := []struct { title string want string }{ {"Pest Control Services", "pest control services"}, {" RRE-01284 - For the provision ", "rre 01284 for the provision"}, {"\u200b\u200bThe supply 3 kg of Li₂TiO₃", "the supply 3 kg of li₂tio₃"}, {"Children & Young People's", "children young people s"}, {"---", ""}, } for _, tt := range tests { if got := normaliseTitle(tt.title); got != tt.want { t.Errorf("normaliseTitle(%q) = %q; want %q", tt.title, got, tt.want) } } } func TestTitleSimilarity(t *testing.T) { tests := []struct { first, second string want float64 }{ {"pest control services", "pest control services", 1}, {"abcdefghij", "abcdefghix", 0.9}, {"abcdefghij", "abcdefghxx", 0.8}, {"kitten", "sitting", 1 - 3.0/7}, {"", "", 0}, {"abc", "", 0}, } for _, tt := range tests { got := titleSimilarity(tt.first, tt.second) if got < tt.want-1e-9 || got > tt.want+1e-9 { t.Errorf( "titleSimilarity(%q, %q) = %v; want %v", tt.first, tt.second, got, tt.want, ) } } } func TestSimilarTitles(t *testing.T) { tests := []struct { first, second string want bool }{ {"housing related support", "housing related support", true}, // One edit in ten is exactly 90%. {"abcdefghij", "abcdefghix", true}, {"abcdefghij", "abcdefghxx", false}, {"pest control services", "pest control service", true}, {"alternative provision", "alternative provision for schools", false}, {"", "", false}, } for _, tt := range tests { if got := similarTitles(tt.first, tt.second); got != tt.want { t.Errorf( "similarTitles(%q, %q) = %v; want %v", tt.first, tt.second, got, tt.want, ) } } } // similarTitles' banded, early-stopping distance agrees with the full one. func TestSimilarTitlesAgreesWithTitleSimilarity(t *testing.T) { //nolint:gosec // G404: seeded math/rand keeps the test deterministic rng := rand.New(rand.NewSource(1)) alphabet := []rune("abc d") random := func() string { runes := make([]rune, 5+rng.Intn(30)) for i := range runes { runes[i] = alphabet[rng.Intn(len(alphabet))] } return string(runes) } mutate := func(title string) string { runes := []rune(title) for range rng.Intn(5) { i := rng.Intn(len(runes)) switch rng.Intn(3) { case 0: runes[i] = alphabet[rng.Intn(len(alphabet))] case 1: runes = append(runes[:i], runes[i+1:]...) default: runes = append(runes[:i], append([]rune{'x'}, runes[i:]...)...) } if len(runes) == 0 { runes = []rune("a") } } return string(runes) } for range 5000 { first := random() second := mutate(first) if rng.Intn(4) == 0 { second = random() } want := titleSimilarity(first, second) >= titleMatchThreshold-1e-9 if got := similarTitles(first, second); got != want { t.Fatalf( "similarTitles(%q, %q) = %v; titleSimilarity = %v", first, second, got, titleSimilarity(first, second), ) } } } func TestBuyerSimilarity(t *testing.T) { tests := []struct { first, second string want float64 known bool }{ {"Jisc", "JISC SERVICES LIMITED", 1, true}, {"The Borough Council of Calderdale", "Borough Council of Calderdale", 1, true}, {"Wakefield Council", "The Council of the City of Wakefield", 1, true}, {"City of York Council", "York and North Yorkshire Combined Authority", 1, true}, {"UK Atomic Energy Authority", "UK Atomic Energy Authoritys", 1, true}, {"Yorkshire Housing Limited", "Anchor Hanover Group", 0, true}, // Only the shorter name's words count, so a name within another // matches it, rightly for York above, wrongly here: the title and // date must match too, so this is accepted. {"North East Lincolnshire", "North Lincolnshire Council", 1, true}, {"Hull City Council", "East Riding of Yorkshire Council", 0, true}, {"The Council", "Jisc", 0, false}, {"", "Jisc", 0, false}, } for _, tt := range tests { got, known := buyerSimilarity(tt.first, tt.second) if known != tt.known || got < tt.want-1e-9 || got > tt.want+1e-9 { t.Errorf( "buyerSimilarity(%q, %q) = %v, %v; want %v, %v", tt.first, tt.second, got, known, tt.want, tt.known, ) } } } func TestDescriptionSimilarity(t *testing.T) { tests := []struct { first, second string want float64 known bool }{ {"Supply of pest control.", "Pest control: the supply", 1, true}, {"Building materials for repairs", "Building works", 0.5, true}, {"", "Anything at all", 0, false}, {"a to be", "Anything at all", 0, false}, } for _, tt := range tests { got, known := descriptionSimilarity(tt.first, tt.second) if known != tt.known || got < tt.want-1e-9 || got > tt.want+1e-9 { t.Errorf( "descriptionSimilarity(%q, %q) = %v, %v; want %v, %v", tt.first, tt.second, got, known, tt.want, tt.known, ) } } } // A title match is confirmed by the buyer or, failing that, the // description; with neither to compare, it isn't. func TestConfirmedRepeat(t *testing.T) { opportunity := func(buyer, description string) *types.Opportunity { return &types.Opportunity{ BuyerName: buyer, Description: description, } } tests := []struct { name string first, last *types.Opportunity want bool }{ {"buyer", opportunity("Jisc", ""), opportunity("JISC LIMITED", ""), true}, { "description, not buyer", opportunity("Echelon", "building materials supply"), opportunity("Fairhive", "supply of building materials"), true, }, { "neither", opportunity("Yorkshire Housing", "pest control in Leeds homes"), opportunity("Anchor Hanover", "pest control for retirement housing"), false, }, {"nothing to compare", opportunity("", ""), opportunity("", ""), false}, } for _, tt := range tests { if got := confirmedRepeat(tt.first, tt.last); got != tt.want { t.Errorf("%s: confirmedRepeat = %v; want %v", tt.name, got, tt.want) } } }