// Package skiprules reads the rules that mark newly fetched opportunities // Skipped, and matches opportunities against them. See README.md for the // file's format. package skiprules import ( "encoding/json" "errors" "fmt" "io" "os" "slices" "strings" "unicode" "git.avsdev.uk/Go/job-search/internal/types" ) // Rule types, the values of a rule's rule_type. const ( // RuleTypeBuyer matches the buyer's ID. RuleTypeBuyer = "buyer" // RuleTypeTitle matches words in the title. RuleTypeTitle = "title" // RuleTypeCPV matches the opportunity's CPV codes. RuleTypeCPV = "cpv" ) // Logic modes, the values of a filter's logic_mode. const ( // LogicModeOr makes a filter match when any of its rules matches. It is // the default. LogicModeOr = "or" // LogicModeAnd makes a filter match only when all of its rules match. LogicModeAnd = "and" ) // Rules are the filters of a rules file, in file order. Create them with // Parse or Load. A nil *Rules has no filters and matches nothing. type Rules struct { filters []filter } // filter is one parsed filter. all is set by logic_mode "and". type filter struct { comment string active bool all bool rules []rule } // rule is one parsed rule. Only the fields for its ruleType are set: values // for a buyer rule, words (lowercase) for a title rule, codes (without check // digits), exact and exclusive for a CPV rule. When exact is false, codes // are prefixes (see cpvPrefix). type rule struct { ruleType string values []string words []string codes []string exact bool exclusive bool } // fileFilter is a filter as it is read from the file. Pointers and nil // slices tell a missing field from a zero one. type fileFilter struct { Comment *string `json:"comment"` Active *bool `json:"active"` LogicMode *string `json:"logic_mode"` Rules []fileRule `json:"rules"` } // fileRule is a rule as it is read from the file. type fileRule struct { RuleType *string `json:"rule_type"` Values []string `json:"values"` Exact *bool `json:"exact"` Exclusive *bool `json:"exclusive"` } // Load reads and parses the rules file at path (see Parse). func Load(path string) (*Rules, error) { file, err := os.Open(path) //nolint:gosec // G304: the configured rules file if err != nil { return nil, err } defer file.Close() //nolint:errcheck // read only rules, err := Parse(file) if err != nil { return nil, fmt.Errorf("skip rules %s: %w", path, err) } return rules, nil } // Parse reads a rules file: a JSON list of filters. Unknown fields, missing // fields (any but logic_mode, exact and exclusive), empty lists of rules or // values, unknown rule types or logic modes, exact or exclusive on a rule // that isn't a CPV rule, // and CPV codes that aren't 8 digits are all refused, naming the filter. func Parse(reader io.Reader) (*Rules, error) { decoder := json.NewDecoder(reader) decoder.DisallowUnknownFields() var fileFilters []fileFilter if err := decoder.Decode(&fileFilters); err != nil { return nil, fmt.Errorf("reading JSON: %w", err) } if fileFilters == nil { return nil, errors.New("skiprules: the file must be a list of filters") } if !errors.Is(decoder.Decode(&struct{}{}), io.EOF) { return nil, errors.New( "skiprules: unexpected content after the list of filters", ) } rules := &Rules{ filters: make([]filter, 0, len(fileFilters)), } for i, fileFilter := range fileFilters { parsed, err := fileFilter.parse() if err != nil { return nil, fmt.Errorf( "filter %d%s: %w", i+1, describe(fileFilter.Comment), err, ) } rules.filters = append(rules.filters, parsed) } return rules, nil } // describe returns comment quoted after a space, to name a filter in an // error, or "" if it has none. func describe(comment *string) string { if comment == nil || *comment == "" { return "" } return fmt.Sprintf(" (%q)", *comment) } // parse checks a filter as read from the file and returns it parsed. func (f fileFilter) parse() (filter, error) { switch { case f.Comment == nil: return filter{}, errors.New("skiprules: comment is required") case f.Active == nil: return filter{}, errors.New("skiprules: active is required") case len(f.Rules) == 0: return filter{}, errors.New( "skiprules: rules is required and must not be empty", ) } logicMode := LogicModeOr if f.LogicMode != nil { logicMode = *f.LogicMode } if logicMode != LogicModeOr && logicMode != LogicModeAnd { return filter{}, fmt.Errorf( "skiprules: logic_mode %q must be %q or %q", logicMode, LogicModeOr, LogicModeAnd, ) } parsed := filter{ comment: *f.Comment, active: *f.Active, all: logicMode == LogicModeAnd, rules: make([]rule, 0, len(f.Rules)), } for i, fileRule := range f.Rules { parsedRule, err := fileRule.parse() if err != nil { return filter{}, fmt.Errorf("rule %d: %w", i+1, err) } parsed.rules = append(parsed.rules, parsedRule) } return parsed, nil } // parse checks a rule as read from the file and returns it parsed. func (r fileRule) parse() (rule, error) { switch { case r.RuleType == nil: return rule{}, errors.New("skiprules: rule_type is required") case len(r.Values) == 0: return rule{}, errors.New( "skiprules: values is required and must not be empty", ) case r.Exact != nil && *r.RuleType != RuleTypeCPV: return rule{}, fmt.Errorf( "skiprules: exact is only for %q rules", RuleTypeCPV, ) case r.Exclusive != nil && *r.RuleType != RuleTypeCPV: return rule{}, fmt.Errorf( "skiprules: exclusive is only for %q rules", RuleTypeCPV, ) } parsed := rule{ ruleType: *r.RuleType, } switch parsed.ruleType { case RuleTypeBuyer: parsed.values = r.Values case RuleTypeTitle: for _, value := range r.Values { parsed.words = append(parsed.words, words(value)...) } if len(parsed.words) == 0 { return rule{}, errors.New("skiprules: values has no words") } case RuleTypeCPV: return r.parseCPV() default: return rule{}, fmt.Errorf( "skiprules: rule_type %q must be %q, %q or %q", parsed.ruleType, RuleTypeBuyer, RuleTypeTitle, RuleTypeCPV, ) } return parsed, nil } // parseCPV returns a CPV rule parsed: exact and exclusive default to true, // and each value must be a CPV code. func (r fileRule) parseCPV() (rule, error) { parsed := rule{ ruleType: RuleTypeCPV, exact: r.Exact == nil || *r.Exact, exclusive: r.Exclusive == nil || *r.Exclusive, } for _, value := range r.Values { code := cpvCode(value) if !isCPVCode(code) { return rule{}, fmt.Errorf( "skiprules: %q is not a CPV code (8 digits, optionally with a check digit)", value, ) } if !parsed.exact { code = cpvPrefix(code) } parsed.codes = append(parsed.codes, code) } return parsed, nil } // Filters returns the number of filters, and how many of them are active. func (r *Rules) Filters() (count, active int) { if r == nil { return 0, 0 } for _, entry := range r.filters { if entry.active { active++ } } return len(r.filters), active } // Match reports whether opportunity should be skipped: whether any active // filter matches it. If one does, it also returns the first such filter's // comment. func (r *Rules) Match(opportunity *types.Opportunity) (string, bool) { if r == nil { return "", false } for _, entry := range r.filters { if entry.matches(opportunity) { return entry.comment, true } } return "", false } // matches reports whether the filter is active and its rules match // opportunity: all of them with logic_mode "and", otherwise any. func (f filter) matches(opportunity *types.Opportunity) bool { if !f.active { return false } // The first rule that doesn't match settles "and" (false), and the // first that does settles "or" (true); otherwise every rule agreed. for _, entry := range f.rules { matched := entry.matches(opportunity) if matched != f.all { return matched } } return f.all } // matches reports whether the rule matches opportunity: for a buyer rule, // its buyer ID is one of the values; for a title rule, every word is a word // of its title; for a CPV rule, see matchesCPV. func (r rule) matches(opportunity *types.Opportunity) bool { switch r.ruleType { case RuleTypeBuyer: return slices.Contains(r.values, opportunity.BuyerID) case RuleTypeTitle: title := words(opportunity.Title) for _, word := range r.words { if !slices.Contains(title, word) { return false } } return true case RuleTypeCPV: return r.matchesCPV(opportunity) } return false } // matchesCPV reports whether a CPV rule matches opportunity: it has CPV // codes, and the rule's codes cover every one of them if the rule is // exclusive, or any one of them if it isn't. func (r rule) matchesCPV(opportunity *types.Opportunity) bool { found := false for _, classification := range opportunity.Classifications { if !classification.IsCPV { continue } found = true // The first code not covered settles an exclusive rule (false), and // the first code covered settles one that isn't (true). covered := r.coversCPV(cpvCode(classification.Code)) if covered != r.exclusive { return covered } } return found && r.exclusive } // coversCPV reports whether code (without its check digit) is one of the // rule's codes or, unless the rule is exact, starts with one of them. func (r rule) coversCPV(code string) bool { if r.exact { return slices.Contains(r.codes, code) } for _, prefix := range r.codes { if strings.HasPrefix(code, prefix) { return true } } return false } // words returns the lowercase words of text. Anything but a letter or a // digit separates words, so "Road-repairs, phase 2" is road, repairs, phase // and 2. func words(text string) []string { return strings.FieldsFunc(strings.ToLower(text), func(char rune) bool { return !unicode.IsLetter(char) && !unicode.IsDigit(char) }) } // cpvCode returns a CPV code without its check digit: "72000000-5" is // "72000000". func cpvCode(code string) string { code, _, _ = strings.Cut(strings.TrimSpace(code), "-") return code } // isCPVCode reports whether code (without its check digit) is 8 digits. func isCPVCode(code string) bool { if len(code) != 8 { return false } for _, char := range code { if char < '0' || char > '9' { return false } } return true } // cpvPrefix returns the prefix a CPV code stands for in a rule that isn't // exact: the code without its trailing zeros, but at least its first two // digits (its division), so "72000000" is "72", "72200000" is "722" and // "70000000" is "70", not "7". func cpvPrefix(code string) string { prefix := strings.TrimRight(code, "0") if len(prefix) < 2 { return code[:2] } return prefix }