Added skip rules, which mark new opportunities Skipped when they're fetched

sources.skip_rules_file names a JSON file of filters, each with a
comment, an active flag, a logic mode ("or" by default, or "and") and
rules by buyer, title words or CPV codes. The fetcher applies them
before saving, so only new opportunities are marked; the marks of
stored ones are never touched. CPV rules match when the opportunity's
codes are all in the list, or any of them with exclusive false, and
by prefix with exact false.

The file is read again when its time or size changes: an invalid edit
is logged and the last good rules kept, and a deleted file means no
rules until it's back. fetcher service and serve refuse to start if it
is define but missing or invalid.

job-search skip-rules check validates a rules file, and skip-rules
apply marks the stored opportunities that match Skipped, or only lists
them with --dry-run.

internal/fetcher/fetcher.go

@@ -1,6 +1,7 @@
 package fetcher
 
 import (
+	"cmp"
 	"context"
 	"errors"
 	"fmt"
@@ -10,6 +11,7 @@ import (
 
 	"git.avsdev.uk/Go/job-search/internal/config"
 	"git.avsdev.uk/Go/job-search/internal/events"
+	"git.avsdev.uk/Go/job-search/internal/skiprules"
 	"git.avsdev.uk/Go/job-search/internal/store"
 	"git.avsdev.uk/Go/job-search/internal/types"
 	"git.avsdev.uk/Go/job-search/internal/util"
@@ -49,6 +51,15 @@ func WithEvents(bus *events.Bus) Option {
 	}
 }
 
+// WithSkipRules makes the workers mark the new opportunities that match the
+// rules in rulesFile Skipped as they save them (see Fetcher.save). The file
+// is checked for changes before each save.
+func WithSkipRules(rulesFile *skiprules.File) Option {
+	return func(f *Fetcher) {
+		f.skipRules = rulesFile
+	}
+}
+
 // saveFunc saves the opportunities a worker fetched; see Fetcher.save.
 type saveFunc func(
 	ctx context.Context,
@@ -81,6 +92,9 @@ type Fetcher struct {
 	events  *events.Bus  // nil: nothing is published
 	started time.Time
 
+	// skipRules mark new opportunities Skipped; nil: none are.
+	skipRules *skiprules.File
+
 	// backgroundCtx is the context the workers run under. backgroundCancel
 	// stops them: Run calls it when its ctx is cancelled, when a worker
 	// fails, and as it returns.
@@ -216,6 +230,10 @@ func (f *Fetcher) startFrom(
 // and publishes it (see WithEvents). Without a store it does nothing. A
 // failure fails the worker's run, so its checkpoint isn't moved on and the
 // next run fetches them again.
+//
+// Opportunities that match the skip rules (see WithSkipRules) are marked
+// Skipped first. The store keeps the marks of an opportunity it already
+// has, so only new ones are skipped.
 func (f *Fetcher) save(
 	ctx context.Context,
 	logger *slog.Logger,
@@ -225,14 +243,30 @@ func (f *Fetcher) save(
 		return nil
 	}
 
+	filters := f.markSkipped(opportunities)
+
 	result, err := f.store.SaveOpportunities(ctx, opportunities)
 	if err != nil {
 		return fmt.Errorf("saving opportunities: %w", err)
 	}
 
+	skipped := 0
+	for i, saved := range result.OpportunityResults {
+		if !saved.NewOpportunity || filters[i] == "" {
+			continue
+		}
+		skipped++
+		tagged(logger).Debug(
+			"skipped new opportunity",
+			"uuid", opportunities[i].UUID,
+			"filter", filters[i],
+		)
+	}
+
 	tagged(logger).Info(
 		"saved opportunities",
 		"new_opportunities", result.NewOpportunities,
+		"skipped_opportunities", skipped,
 		"updated_opportunities", result.UpdatedOpportunities,
 		"new_releases", result.NewReleases,
 	)
@@ -243,6 +277,30 @@ func (f *Fetcher) save(
 	return nil
 }
 
+// markSkipped marks the opportunities that match the skip rules Skipped,
+// and returns the comment of the filter each matched, indexed like
+// opportunities: "" for one that matched none (every one, with no skip
+// rules).
+func (f *Fetcher) markSkipped(opportunities []*types.Opportunity) []string {
+	filters := make([]string, len(opportunities))
+	if f.skipRules == nil {
+		return filters
+	}
+
+	rules := f.skipRules.Rules()
+	for i, opportunity := range opportunities {
+		comment, matched := rules.Match(opportunity)
+		if !matched {
+			continue
+		}
+		opportunity.Marks.Skipped = true
+		// A filter's comment may be empty; it still names one here.
+		filters[i] = cmp.Or(comment, "(no comment)")
+	}
+
+	return filters
+}
+
 // publishClassificationChanges publishes an events.ClassificationsUpdated
 // event for the classification codes a save, with result, brought into or
 // out of use, if it changed any and a bus was given WithEvents. It's

internal/fetcher/fetcher_test.go

@@ -18,6 +18,7 @@ import (
 
 	"git.avsdev.uk/Go/job-search/internal/config"
 	"git.avsdev.uk/Go/job-search/internal/events"
+	"git.avsdev.uk/Go/job-search/internal/skiprules"
 	"git.avsdev.uk/Go/job-search/internal/store"
 	"git.avsdev.uk/Go/job-search/internal/types"
 	"git.avsdev.uk/Go/job-search/internal/util"
@@ -876,3 +877,80 @@ func TestSavePublishesClassificationChanges(t *testing.T) {
 		)
 	}
 }
+
+// skipRulesFile writes rules to a file and opens it.
+func skipRulesFile(t *testing.T, rules string) *skiprules.File {
+	t.Helper()
+	path := t.TempDir() + "/skip-rules.json"
+	if err := os.WriteFile(path, []byte(rules), 0o600); err != nil {
+		t.Fatal(err)
+	}
+	file, err := skiprules.Open(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return file
+}
+
+// A new opportunity that matches the skip rules is saved Skipped; one
+// already stored keeps its marks.
+func TestSaveSkipsNewOpportunities(t *testing.T) {
+	rules := skipRulesFile(t, `[{"comment": "IT", "active": true,
+		"rules": [{"rule_type": "cpv", "values": ["72000000"]}]}]`)
+	ctx := context.Background()
+	skipped := func(s *store.Store) bool {
+		t.Helper()
+		opportunity, err := s.GetOpportunity(ctx, "ocds-1")
+		if err != nil {
+			t.Fatal(err)
+		}
+		return opportunity.Marks.Skipped
+	}
+	matching := withClassification("r1", "2026-10-01T12:00:00Z", "72000000")
+
+	// New and matching: skipped, and counted in the log.
+	var logged bytes.Buffer
+	logger := slog.New(slog.NewTextHandler(&logged, &slog.HandlerOptions{
+		Level: slog.LevelDebug,
+	}))
+	s := openTestStore(t)
+	f := New(&config.JSCSources{}, WithStore(s), WithSkipRules(rules))
+	if err := f.save(ctx, logger, fetched(t, matching)); err != nil {
+		t.Fatal(err)
+	}
+	if !skipped(s) {
+		t.Error("a new matching opportunity wasn't skipped")
+	}
+	for _, want := range []string{
+		"skipped_opportunities=1",
+		"msg=\"skipped new opportunity\" component=fetcher uuid=ocds-1 filter=IT",
+	} {
+		if !strings.Contains(logged.String(), want) {
+			t.Errorf("log has no %q:\n%s", want, logged.String())
+		}
+	}
+
+	// New and not matching: not skipped.
+	s = openTestStore(t)
+	f = New(&config.JSCSources{}, WithStore(s), WithSkipRules(rules))
+	other := withClassification("r1", "2026-10-01T12:00:00Z", "03000000")
+	if err := f.save(ctx, slog.Default(), fetched(t, other)); err != nil {
+		t.Fatal(err)
+	}
+	if skipped(s) {
+		t.Error("a new opportunity that doesn't match was skipped")
+	}
+
+	// Stored already: a new release that matches doesn't skip it.
+	newRelease := withClassification("r2", "2026-10-02T12:00:00Z", "72000000")
+	logged.Reset()
+	if err := f.save(ctx, logger, fetched(t, newRelease)); err != nil {
+		t.Fatal(err)
+	}
+	if skipped(s) {
+		t.Error("a stored opportunity was skipped by a new release")
+	}
+	if !strings.Contains(logged.String(), "skipped_opportunities=0") {
+		t.Errorf("log doesn't count 0 skipped:\n%s", logged.String())
+	}
+}