job-search/internal/fetcher/match_test.go

package fetcher

import (
	"math/rand"
	"testing"

	"git.avsdev.uk/Go/job-search/internal/types"
)

func TestNormaliseTitle(t *testing.T) {
	tests := []struct {
		title string
		want  string
	}{
		{"Pest Control Services", "pest control services"},
		{"  RRE-01284 - For the provision  ", "rre 01284 for the provision"},
		{"\u200b\u200bThe supply 3 kg of Li₂TiO₃", "the supply 3 kg of li₂tio₃"},
		{"Children & Young People's", "children young people s"},
		{"---", ""},
	}
	for _, tt := range tests {
		if got := normaliseTitle(tt.title); got != tt.want {
			t.Errorf("normaliseTitle(%q) = %q; want %q", tt.title, got, tt.want)
		}
	}
}

func TestTitleSimilarity(t *testing.T) {
	tests := []struct {
		first, second string
		want          float64
	}{
		{"pest control services", "pest control services", 1},
		{"abcdefghij", "abcdefghix", 0.9},
		{"abcdefghij", "abcdefghxx", 0.8},
		{"kitten", "sitting", 1 - 3.0/7},
		{"", "", 0},
		{"abc", "", 0},
	}
	for _, tt := range tests {
		got := titleSimilarity(tt.first, tt.second)
		if got < tt.want-1e-9 || got > tt.want+1e-9 {
			t.Errorf(
				"titleSimilarity(%q, %q) = %v; want %v",
				tt.first, tt.second, got, tt.want,
			)
		}
	}
}

func TestSimilarTitles(t *testing.T) {
	tests := []struct {
		first, second string
		want          bool
	}{
		{"housing related support", "housing related support", true},
		// One edit in ten is exactly 90%.
		{"abcdefghij", "abcdefghix", true},
		{"abcdefghij", "abcdefghxx", false},
		{"pest control services", "pest control service", true},
		{"alternative provision", "alternative provision for schools", false},
		{"", "", false},
	}
	for _, tt := range tests {
		if got := similarTitles(tt.first, tt.second); got != tt.want {
			t.Errorf(
				"similarTitles(%q, %q) = %v; want %v",
				tt.first, tt.second, got, tt.want,
			)
		}
	}
}

// similarTitles' banded, early-stopping distance agrees with the full one.
func TestSimilarTitlesAgreesWithTitleSimilarity(t *testing.T) {
	//nolint:gosec // G404: seeded math/rand keeps the test deterministic
	rng := rand.New(rand.NewSource(1))
	alphabet := []rune("abc d")
	random := func() string {
		runes := make([]rune, 5+rng.Intn(30))
		for i := range runes {
			runes[i] = alphabet[rng.Intn(len(alphabet))]
		}
		return string(runes)
	}
	mutate := func(title string) string {
		runes := []rune(title)
		for range rng.Intn(5) {
			i := rng.Intn(len(runes))
			switch rng.Intn(3) {
			case 0:
				runes[i] = alphabet[rng.Intn(len(alphabet))]
			case 1:
				runes = append(runes[:i], runes[i+1:]...)
			default:
				runes = append(runes[:i], append([]rune{'x'}, runes[i:]...)...)
			}
			if len(runes) == 0 {
				runes = []rune("a")
			}
		}
		return string(runes)
	}

	for range 5000 {
		first := random()
		second := mutate(first)
		if rng.Intn(4) == 0 {
			second = random()
		}
		want := titleSimilarity(first, second) >= titleMatchThreshold-1e-9
		if got := similarTitles(first, second); got != want {
			t.Fatalf(
				"similarTitles(%q, %q) = %v; titleSimilarity = %v",
				first, second, got, titleSimilarity(first, second),
			)
		}
	}
}

func TestBuyerSimilarity(t *testing.T) {
	tests := []struct {
		first, second string
		want          float64
		known         bool
	}{
		{"Jisc", "JISC SERVICES LIMITED", 1, true},
		{"The Borough Council of Calderdale", "Borough Council of Calderdale", 1, true},
		{"Wakefield Council", "The Council of the City of Wakefield", 1, true},
		{"City of York Council", "York and North Yorkshire Combined Authority", 1, true},
		{"UK Atomic Energy Authority", "UK Atomic Energy Authoritys", 1, true},
		{"Yorkshire Housing Limited", "Anchor Hanover Group", 0, true},
		// Only the shorter name's words count, so a name within another
		// matches it, rightly for York above, wrongly here: the title and
		// date must match too, so this is accepted.
		{"North East Lincolnshire", "North Lincolnshire Council", 1, true},
		{"Hull City Council", "East Riding of Yorkshire Council", 0, true},
		{"The Council", "Jisc", 0, false},
		{"", "Jisc", 0, false},
	}
	for _, tt := range tests {
		got, known := buyerSimilarity(tt.first, tt.second)
		if known != tt.known || got < tt.want-1e-9 || got > tt.want+1e-9 {
			t.Errorf(
				"buyerSimilarity(%q, %q) = %v, %v; want %v, %v",
				tt.first, tt.second, got, known, tt.want, tt.known,
			)
		}
	}
}

func TestDescriptionSimilarity(t *testing.T) {
	tests := []struct {
		first, second string
		want          float64
		known         bool
	}{
		{"Supply of pest control.", "Pest control: the supply", 1, true},
		{"Building materials for repairs", "Building works", 0.5, true},
		{"", "Anything at all", 0, false},
		{"a to be", "Anything at all", 0, false},
	}
	for _, tt := range tests {
		got, known := descriptionSimilarity(tt.first, tt.second)
		if known != tt.known || got < tt.want-1e-9 || got > tt.want+1e-9 {
			t.Errorf(
				"descriptionSimilarity(%q, %q) = %v, %v; want %v, %v",
				tt.first, tt.second, got, known, tt.want, tt.known,
			)
		}
	}
}

// A title match is confirmed by the buyer or, failing that, the
// description; with neither to compare, it isn't.
func TestConfirmedRepeat(t *testing.T) {
	opportunity := func(buyer, description string) *types.Opportunity {
		return &types.Opportunity{
			BuyerName:   buyer,
			Description: description,
		}
	}
	tests := []struct {
		name        string
		first, last *types.Opportunity
		want        bool
	}{
		{"buyer", opportunity("Jisc", ""), opportunity("JISC LIMITED", ""), true},
		{
			"description, not buyer",
			opportunity("Echelon", "building materials supply"),
			opportunity("Fairhive", "supply of building materials"),
			true,
		},
		{
			"neither",
			opportunity("Yorkshire Housing", "pest control in Leeds homes"),
			opportunity("Anchor Hanover", "pest control for retirement housing"),
			false,
		},
		{"nothing to compare", opportunity("", ""), opportunity("", ""), false},
	}
	for _, tt := range tests {
		if got := confirmedRepeat(tt.first, tt.last); got != tt.want {
			t.Errorf("%s: confirmedRepeat = %v; want %v", tt.name, got, tt.want)
		}
	}
}