package fetcher import ( "math" "strings" "unicode" "git.avsdev.uk/Go/job-search/internal/types" ) // titleMatchThreshold is how alike two titles must be (see titleSimilarity) // for an EU Supply tender to be taken as one already stored from another // source, such as the Find a Tender notice the portal publishes for most of // its tenders (the user's rule: a 90% match). const titleMatchThreshold = 0.9 // The other thresholds of a match (see confirmedRepeat). The user's rule // (2026-10-21): a title match must also be backed by the buyer or the // description, at a ratio of Claude's choosing; 0.8 kept every real repeat // in a live run that either could check, and rejected the one false match. const ( // buyerMatchThreshold is how much of the shorter buyer name's // significant words must be in the other's (see buyerSimilarity). buyerMatchThreshold = 0.8 // descriptionMatchThreshold is how much of the shorter description's // words must be in the other's (see descriptionSimilarity). descriptionMatchThreshold = 0.8 // wordMatchThreshold is how alike two words of buyer names must be to // count as the same word, so a typo or a plural still matches. wordMatchThreshold = 0.8 // minDescriptionWordLength is the shortest word a description is // compared by; shorter ones ("the", "and", "for") say little. minDescriptionWordLength = 4 ) // genericBuyerWords are left out of buyer names before comparing them: // legal forms and kinds of body, which the two sites give differently // ("Jisc" and "JISC SERVICES LIMITED", "The Borough Council of Calderdale" // and "Borough Council of Calderdale") and which say nothing about which // buyer it is. var genericBuyerWords = map[string]bool{ "the": true, "of": true, "and": true, "for": true, "limited": true, "ltd": true, "plc": true, "llp": true, "cic": true, "council": true, "borough": true, "metropolitan": true, "city": true, "county": true, "district": true, "authority": true, "services": true, "group": true, "trust": true, } // confirmedRepeat reports whether other, whose title already matches // opportunity's (see similarTitles), is the same tender: their buyers or // their descriptions must match too. Generic titles ("Pest control // services") are bought by many buyers, so the title alone isn't enough. func confirmedRepeat(opportunity, other *types.Opportunity) bool { buyer, buyerKnown := buyerSimilarity( opportunity.BuyerName, other.BuyerName, ) if buyerKnown && buyer >= buyerMatchThreshold { return true } description, descriptionKnown := descriptionSimilarity( opportunity.Description, other.Description, ) return descriptionKnown && description >= descriptionMatchThreshold } // buyerSimilarity returns the share of the shorter buyer name's significant // words (all but genericBuyerWords) that are in the other name, each at // least wordMatchThreshold alike to one of its words. It is false if either // name has no significant words, when the buyers can't be compared. func buyerSimilarity(first, second string) (float64, bool) { firstWords := significantWords(first) secondWords := significantWords(second) if len(firstWords) == 0 || len(secondWords) == 0 { return 0, false } if len(secondWords) < len(firstWords) { firstWords, secondWords = secondWords, firstWords } found := 0 for _, word := range firstWords { for _, other := range secondWords { if similarWords(word, other) { found++ break } } } return float64(found) / float64(len(firstWords)), true } // significantWords returns the words of a buyer name, normalised as titles // are, without genericBuyerWords. func significantWords(name string) []string { var words []string for word := range strings.FieldsSeq(normaliseTitle(name)) { if !genericBuyerWords[word] { words = append(words, word) } } return words } // similarWords reports whether two words are at least wordMatchThreshold // alike (see titleSimilarity). func similarWords(first, second string) bool { firstRunes, secondRunes := []rune(first), []rune(second) longest := max(len(firstRunes), len(secondRunes)) limit := int(math.Floor((1-wordMatchThreshold)*float64(longest) + 1e-9)) _, within := editDistance(firstRunes, secondRunes, limit) return within } // descriptionSimilarity returns the share of the shorter description's // distinct words (of minDescriptionWordLength letters or more) that are in // the other. It is false if either has no such words: EU Supply leaves // some descriptions empty. func descriptionSimilarity(first, second string) (float64, bool) { firstWords := descriptionWords(first) secondWords := descriptionWords(second) if len(firstWords) == 0 || len(secondWords) == 0 { return 0, false } if len(secondWords) < len(firstWords) { firstWords, secondWords = secondWords, firstWords } found := 0 for word := range firstWords { if secondWords[word] { found++ } } return float64(found) / float64(len(firstWords)), true } // descriptionWords returns the distinct words of a description, normalised // as titles are, of minDescriptionWordLength letters or more. func descriptionWords(description string) map[string]bool { words := make(map[string]bool) for word := range strings.FieldsSeq(normaliseTitle(description)) { if len([]rune(word)) >= minDescriptionWordLength { words[word] = true } } return words } // normaliseTitle returns title as titles are compared: lower case, with // each run of anything but letters and numbers (punctuation, white space, // zero width spaces) made one space, and trimmed. func normaliseTitle(title string) string { var normalised strings.Builder space := false for _, character := range strings.ToLower(title) { if unicode.IsLetter(character) || unicode.IsNumber(character) { if space && normalised.Len() > 0 { normalised.WriteByte(' ') } normalised.WriteRune(character) space = false continue } space = true } return normalised.String() } // titleSimilarity returns how alike two normalised titles (see // normaliseTitle) are, from 0 to 1: one less their edit distance (the // characters inserted, deleted or changed to turn one into the other) over // the longer one's length. Two empty titles are not alike (0). func titleSimilarity(first, second string) float64 { firstRunes, secondRunes := []rune(first), []rune(second) longest := max(len(firstRunes), len(secondRunes)) if longest == 0 { return 0 } distance, _ := editDistance(firstRunes, secondRunes, longest) return 1 - float64(distance)/float64(longest) } // similarTitles reports whether two normalised titles are at least // titleMatchThreshold alike. It gives up as soon as the distance can't be // small enough, so it is much faster than titleSimilarity for titles that // aren't. func similarTitles(first, second string) bool { firstRunes, secondRunes := []rune(first), []rune(second) longest := max(len(firstRunes), len(secondRunes)) if longest == 0 { return false } // The epsilon stops (1-0.9)*10 = 0.99999... rounding down to 0. limit := int(math.Floor((1-titleMatchThreshold)*float64(longest) + 1e-9)) _, within := editDistance(firstRunes, secondRunes, limit) return within } // editDistance returns the Levenshtein distance between source and target, // and whether it is at most limit. Only the cells within limit of the // diagonal are computed, and it stops once a whole row exceeds limit, when // the distance returned is limit+1. func editDistance(source, target []rune, limit int) (int, bool) { if abs(len(source)-len(target)) > limit { return limit + 1, false } exceeded := limit + 1 previous := make([]int, len(target)+1) current := make([]int, len(target)+1) for column := range previous { previous[column] = min(column, exceeded) } for row := 1; row <= len(source); row++ { low, high := max(1, row-limit), min(len(target), row+limit) current[0] = min(row, exceeded) if low > 1 { current[low-1] = exceeded } rowMin := current[0] for column := low; column <= high; column++ { cost := 1 if source[row-1] == target[column-1] { cost = 0 } value := min( previous[column]+1, current[column-1]+1, previous[column-1]+cost, ) current[column] = min(value, exceeded) rowMin = min(rowMin, current[column]) } if high < len(target) { current[high+1] = exceeded } if rowMin > limit { return exceeded, false } previous, current = current, previous } distance := previous[len(target)] return distance, distance <= limit } // abs returns the absolute value of value. func abs(value int) int { if value < 0 { return -value } return value }