job-search/internal/fetcher/match.go

package fetcher

import (
	"math"
	"strings"
	"unicode"

	"git.avsdev.uk/Go/job-search/internal/types"
)

// titleMatchThreshold is how alike two titles must be (see titleSimilarity)
// for an EU Supply tender to be taken as one already stored from another
// source, such as the Find a Tender notice the portal publishes for most of
// its tenders (the user's rule: a 90% match).
const titleMatchThreshold = 0.9

// The other thresholds of a match (see confirmedRepeat). The user's rule
// (2026-10-21): a title match must also be backed by the buyer or the
// description, at a ratio of Claude's choosing; 0.8 kept every real repeat
// in a live run that either could check, and rejected the one false match.
const (
	// buyerMatchThreshold is how much of the shorter buyer name's
	// significant words must be in the other's (see buyerSimilarity).
	buyerMatchThreshold = 0.8
	// descriptionMatchThreshold is how much of the shorter description's
	// words must be in the other's (see descriptionSimilarity).
	descriptionMatchThreshold = 0.8
	// wordMatchThreshold is how alike two words of buyer names must be to
	// count as the same word, so a typo or a plural still matches.
	wordMatchThreshold = 0.8
	// minDescriptionWordLength is the shortest word a description is
	// compared by; shorter ones ("the", "and", "for") say little.
	minDescriptionWordLength = 4
)

// genericBuyerWords are left out of buyer names before comparing them:
// legal forms and kinds of body, which the two sites give differently
// ("Jisc" and "JISC SERVICES LIMITED", "The Borough Council of Calderdale"
// and "Borough Council of Calderdale") and which say nothing about which
// buyer it is.
var genericBuyerWords = map[string]bool{
	"the": true, "of": true, "and": true, "for": true,
	"limited": true, "ltd": true, "plc": true, "llp": true, "cic": true,
	"council": true, "borough": true, "metropolitan": true, "city": true,
	"county": true, "district": true, "authority": true, "services": true,
	"group": true, "trust": true,
}

// confirmedRepeat reports whether other, whose title already matches
// opportunity's (see similarTitles), is the same tender: their buyers or
// their descriptions must match too. Generic titles ("Pest control
// services") are bought by many buyers, so the title alone isn't enough.
func confirmedRepeat(opportunity, other *types.Opportunity) bool {
	buyer, buyerKnown := buyerSimilarity(
		opportunity.BuyerName, other.BuyerName,
	)
	if buyerKnown && buyer >= buyerMatchThreshold {
		return true
	}
	description, descriptionKnown := descriptionSimilarity(
		opportunity.Description, other.Description,
	)
	return descriptionKnown && description >= descriptionMatchThreshold
}

// buyerSimilarity returns the share of the shorter buyer name's significant
// words (all but genericBuyerWords) that are in the other name, each at
// least wordMatchThreshold alike to one of its words. It is false if either
// name has no significant words, when the buyers can't be compared.
func buyerSimilarity(first, second string) (float64, bool) {
	firstWords := significantWords(first)
	secondWords := significantWords(second)
	if len(firstWords) == 0 || len(secondWords) == 0 {
		return 0, false
	}
	if len(secondWords) < len(firstWords) {
		firstWords, secondWords = secondWords, firstWords
	}

	found := 0
	for _, word := range firstWords {
		for _, other := range secondWords {
			if similarWords(word, other) {
				found++
				break
			}
		}
	}
	return float64(found) / float64(len(firstWords)), true
}

// significantWords returns the words of a buyer name, normalised as titles
// are, without genericBuyerWords.
func significantWords(name string) []string {
	var words []string
	for word := range strings.FieldsSeq(normaliseTitle(name)) {
		if !genericBuyerWords[word] {
			words = append(words, word)
		}
	}
	return words
}

// similarWords reports whether two words are at least wordMatchThreshold
// alike (see titleSimilarity).
func similarWords(first, second string) bool {
	firstRunes, secondRunes := []rune(first), []rune(second)
	longest := max(len(firstRunes), len(secondRunes))
	limit := int(math.Floor((1-wordMatchThreshold)*float64(longest) + 1e-9))
	_, within := editDistance(firstRunes, secondRunes, limit)
	return within
}

// descriptionSimilarity returns the share of the shorter description's
// distinct words (of minDescriptionWordLength letters or more) that are in
// the other. It is false if either has no such words: EU Supply leaves
// some descriptions empty.
func descriptionSimilarity(first, second string) (float64, bool) {
	firstWords := descriptionWords(first)
	secondWords := descriptionWords(second)
	if len(firstWords) == 0 || len(secondWords) == 0 {
		return 0, false
	}
	if len(secondWords) < len(firstWords) {
		firstWords, secondWords = secondWords, firstWords
	}

	found := 0
	for word := range firstWords {
		if secondWords[word] {
			found++
		}
	}
	return float64(found) / float64(len(firstWords)), true
}

// descriptionWords returns the distinct words of a description, normalised
// as titles are, of minDescriptionWordLength letters or more.
func descriptionWords(description string) map[string]bool {
	words := make(map[string]bool)
	for word := range strings.FieldsSeq(normaliseTitle(description)) {
		if len([]rune(word)) >= minDescriptionWordLength {
			words[word] = true
		}
	}
	return words
}

// normaliseTitle returns title as titles are compared: lower case, with
// each run of anything but letters and numbers (punctuation, white space,
// zero width spaces) made one space, and trimmed.
func normaliseTitle(title string) string {
	var normalised strings.Builder
	space := false
	for _, character := range strings.ToLower(title) {
		if unicode.IsLetter(character) || unicode.IsNumber(character) {
			if space && normalised.Len() > 0 {
				normalised.WriteByte(' ')
			}
			normalised.WriteRune(character)
			space = false
			continue
		}
		space = true
	}
	return normalised.String()
}

// titleSimilarity returns how alike two normalised titles (see
// normaliseTitle) are, from 0 to 1: one less their edit distance (the
// characters inserted, deleted or changed to turn one into the other) over
// the longer one's length. Two empty titles are not alike (0).
func titleSimilarity(first, second string) float64 {
	firstRunes, secondRunes := []rune(first), []rune(second)
	longest := max(len(firstRunes), len(secondRunes))
	if longest == 0 {
		return 0
	}
	distance, _ := editDistance(firstRunes, secondRunes, longest)
	return 1 - float64(distance)/float64(longest)
}

// similarTitles reports whether two normalised titles are at least
// titleMatchThreshold alike. It gives up as soon as the distance can't be
// small enough, so it is much faster than titleSimilarity for titles that
// aren't.
func similarTitles(first, second string) bool {
	firstRunes, secondRunes := []rune(first), []rune(second)
	longest := max(len(firstRunes), len(secondRunes))
	if longest == 0 {
		return false
	}
	// The epsilon stops (1-0.9)*10 = 0.99999... rounding down to 0.
	limit := int(math.Floor((1-titleMatchThreshold)*float64(longest) + 1e-9))
	_, within := editDistance(firstRunes, secondRunes, limit)
	return within
}

// editDistance returns the Levenshtein distance between source and target,
// and whether it is at most limit. Only the cells within limit of the
// diagonal are computed, and it stops once a whole row exceeds limit, when
// the distance returned is limit+1.
func editDistance(source, target []rune, limit int) (int, bool) {
	if abs(len(source)-len(target)) > limit {
		return limit + 1, false
	}

	exceeded := limit + 1
	previous := make([]int, len(target)+1)
	current := make([]int, len(target)+1)
	for column := range previous {
		previous[column] = min(column, exceeded)
	}

	for row := 1; row <= len(source); row++ {
		low, high := max(1, row-limit), min(len(target), row+limit)
		current[0] = min(row, exceeded)
		if low > 1 {
			current[low-1] = exceeded
		}
		rowMin := current[0]
		for column := low; column <= high; column++ {
			cost := 1
			if source[row-1] == target[column-1] {
				cost = 0
			}
			value := min(
				previous[column]+1, current[column-1]+1, previous[column-1]+cost,
			)
			current[column] = min(value, exceeded)
			rowMin = min(rowMin, current[column])
		}
		if high < len(target) {
			current[high+1] = exceeded
		}
		if rowMin > limit {
			return exceeded, false
		}
		previous, current = current, previous
	}

	distance := previous[len(target)]
	return distance, distance <= limit
}

// abs returns the absolute value of value.
func abs(value int) int {
	if value < 0 {
		return -value
	}
	return value
}