Added skip rules, which mark new opportunities Skipped when they're fetched

sources.skip_rules_file names a JSON file of filters, each with a
comment, an active flag, a logic mode ("or" by default, or "and") and
rules by buyer, title words or CPV codes. The fetcher applies them
before saving, so only new opportunities are marked; the marks of
stored ones are never touched. CPV rules match when the opportunity's
codes are all in the list, or any of them with exclusive false, and
by prefix with exact false.

The file is read again when its time or size changes: an invalid edit
is logged and the last good rules kept, and a deleted file means no
rules until it's back. fetcher service and serve refuse to start if it
is define but missing or invalid.

job-search skip-rules check validates a rules file, and skip-rules
apply marks the stored opportunities that match Skipped, or only lists
them with --dry-run.

internal/cli/fetch/service.go

@@ -54,7 +54,11 @@ func commandService(opts *cli.Options) *cobra.Command {
 				stop()
 			}()
 
-			var fetcherOptions []fetcher.Option
+			fetcherOptions, err := cli.SkipRulesOptions(conf)
+			if err != nil {
+				return err
+			}
+
 			opportunityStore, openErr := cli.OpenConfiguredStore(ctx, conf)
 			switch {
 			case errors.Is(openErr, cli.ErrNoDatabase):

internal/cli/serve/command.go

@@ -110,7 +110,11 @@ set, the API serves the stored opportunities under /api/opportunities.`,
 
 			// The store is shared by the sources, which save to it, and the API,
 			// which serves it.
-			var fetcherOptions []fetcher.Option
+			fetcherOptions, err := cli.SkipRulesOptions(conf)
+			if err != nil {
+				return err
+			}
+
 			opportunityStore, openErr := cli.OpenConfiguredStore(ctx, conf)
 			switch {
 			case errors.Is(openErr, cli.ErrNoDatabase):

internal/cli/skiprules.go

@@ -0,0 +1,27 @@
+package cli
+
+import (
+	"fmt"
+
+	"git.avsdev.uk/Go/job-search/internal/config"
+	"git.avsdev.uk/Go/job-search/internal/fetcher"
+	"git.avsdev.uk/Go/job-search/internal/skiprules"
+)
+
+// SkipRulesOptions opens the rules file at sources.skip_rules_file and
+// returns the fetcher option that applies it, or none if no file is
+// configured. A missing or invalid file is an error, so a command doesn't
+// start without the rules it was given.
+func SkipRulesOptions(conf *config.JobSearchConfig) ([]fetcher.Option, error) {
+	path := conf.Sources.SkipRulesFile
+	if path == "" {
+		return nil, nil
+	}
+
+	rulesFile, err := skiprules.Open(path)
+	if err != nil {
+		return nil, fmt.Errorf("sources.skip_rules_file: %w", err)
+	}
+
+	return []fetcher.Option{fetcher.WithSkipRules(rulesFile)}, nil
+}

internal/cli/skiprules/apply.go

@@ -0,0 +1,174 @@
+package skiprules
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"log/slog"
+	"text/tabwriter"
+
+	"github.com/spf13/cobra"
+
+	"git.avsdev.uk/Go/job-search/internal/cli"
+	"git.avsdev.uk/Go/job-search/internal/skiprules"
+	"git.avsdev.uk/Go/job-search/internal/store"
+	"git.avsdev.uk/Go/job-search/internal/types"
+)
+
+// applyPageSize is how many opportunities apply reads at a time.
+const applyPageSize = 500
+
+// match is a stored opportunity the rules skip, and the filter that does.
+type match struct {
+	opportunity types.Opportunity
+	filter      string
+}
+
+// commandApply returns the "apply" command, which marks the stored
+// opportunities that match the rules Skipped.
+func commandApply(opts *cli.Options) *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   "apply [file]",
+		Short: "Mark stored opportunities that match skip rules Skipped",
+		Long: `Mark the stored opportunities that match the skip rules Skipped,
+listing each one. The rules are read from the file given, or else from
+sources.skip_rules_file. Only the Skipped mark is ever set, and never
+cleared: opportunities already skipped are left alone. With --dry-run, the
+matches are listed but nothing is changed.
+
+Each change is recorded for clients to catch up from, but isn't pushed to
+open browsers as a running "serve" pushes its own changes.`,
+		Args: cobra.MaximumNArgs(1),
+		PersistentPreRun: func(cmd *cobra.Command, args []string) {
+			slog.Debug("command persistent pre-run", "command", "skip-rules.apply")
+		},
+		PersistentPostRun: func(cmd *cobra.Command, args []string) {
+			slog.Debug("command persistent post-run", "command", "skip-rules.apply")
+		},
+		RunE: func(cmd *cobra.Command, args []string) (err error) {
+			slog.Debug("command run", "command", "skip-rules.apply")
+
+			dryRun, _ := cmd.Flags().GetBool("dry-run")
+
+			conf, err := opts.RunSetup(cmd, true, true)
+			if err != nil {
+				return fmt.Errorf("load configuration: %w", err)
+			}
+
+			rules, _, err := loadRules(conf, args)
+			if err != nil {
+				return err
+			}
+
+			opportunityStore, err := cli.OpenConfiguredStore(cmd.Context(), conf)
+			if err != nil {
+				return err
+			}
+			defer func() {
+				err = errors.Join(err, opportunityStore.Close())
+			}()
+
+			matches, err := findMatches(cmd.Context(), opportunityStore, rules)
+			if err != nil {
+				return err
+			}
+			err = printMatches(cmd.OutOrStdout(), matches)
+			if err != nil {
+				return err
+			}
+
+			if dryRun {
+				_, err = fmt.Fprintf(
+					cmd.OutOrStdout(),
+					"%d opportunities would be skipped (dry run)\n",
+					len(matches),
+				)
+				return err
+			}
+
+			err = skipAll(cmd.Context(), opportunityStore, matches)
+			if err != nil {
+				return err
+			}
+
+			_, err = fmt.Fprintf(
+				cmd.OutOrStdout(), "Skipped %d opportunities\n", len(matches),
+			)
+			return err
+		},
+	}
+	cmd.Flags().Bool("dry-run", false, "list the matches without changing them")
+
+	return cmd
+}
+
+// findMatches returns the stored opportunities not yet skipped that rules
+// match, newest first. They are all read before any is changed, so changing
+// them doesn't move the pages being read.
+func findMatches(
+	ctx context.Context, opportunityStore *store.Store, rules *skiprules.Rules,
+) ([]match, error) {
+	var matches []match
+	for offset := 0; ; offset += applyPageSize {
+		page, err := opportunityStore.ListOpportunities(
+			ctx,
+			store.OpportunityFilter{
+				Skipped: store.MarkUnset,
+				Limit:   applyPageSize,
+				Offset:  offset,
+			},
+		)
+		if err != nil {
+			return nil, fmt.Errorf("listing opportunities: %w", err)
+		}
+
+		for i := range page {
+			filter, matched := rules.Match(&page[i])
+			if matched {
+				matches = append(matches, match{
+					opportunity: page[i],
+					filter:      filter,
+				})
+			}
+		}
+
+		if len(page) < applyPageSize {
+			return matches, nil
+		}
+	}
+}
+
+// printMatches lists matches, one per line: UUID, filter and title.
+func printMatches(out io.Writer, matches []match) error {
+	table := tabwriter.NewWriter(out, 0, 0, 2, ' ', 0)
+	for _, found := range matches {
+		_, err := fmt.Fprintf(
+			table,
+			"%s\t%s\t%s\n",
+			found.opportunity.UUID,
+			found.filter,
+			found.opportunity.Title,
+		)
+		if err != nil {
+			return err
+		}
+	}
+	return table.Flush()
+}
+
+// skipAll sets the Skipped mark of every match, keeping its other marks.
+func skipAll(
+	ctx context.Context, opportunityStore *store.Store, matches []match,
+) error {
+	for _, found := range matches {
+		marks := found.opportunity.Marks
+		marks.Skipped = true
+		if err := opportunityStore.SetMarks(ctx, marks); err != nil {
+			return fmt.Errorf(
+				"skipping %s: %w", found.opportunity.UUID, err,
+			)
+		}
+	}
+	return nil
+}

internal/cli/skiprules/check.go

@@ -0,0 +1,44 @@
+package skiprules
+
+import (
+	"fmt"
+	"log/slog"
+
+	"github.com/spf13/cobra"
+
+	"git.avsdev.uk/Go/job-search/internal/cli"
+)
+
+// commandCheck returns the "check" command, which reports whether a rules
+// file is valid.
+func commandCheck(opts *cli.Options) *cobra.Command {
+	return &cobra.Command{
+		Use:   "check [file]",
+		Short: "Check a skip rules file (default: sources.skip_rules_file)",
+		Args:  cobra.MaximumNArgs(1),
+		PersistentPreRun: func(cmd *cobra.Command, args []string) {
+			slog.Debug("command persistent pre-run", "command", "skip-rules.check")
+		},
+		PersistentPostRun: func(cmd *cobra.Command, args []string) {
+			slog.Debug("command persistent post-run", "command", "skip-rules.check")
+		},
+		RunE: func(cmd *cobra.Command, args []string) error {
+			slog.Debug("command run", "command", "skip-rules.check")
+
+			conf, err := opts.RunSetup(cmd, true, true)
+			if err != nil {
+				return fmt.Errorf("load configuration: %w", err)
+			}
+
+			rules, path, err := loadRules(conf, args)
+			if err != nil {
+				return err
+			}
+
+			_, err = fmt.Fprintf(
+				cmd.OutOrStdout(), "%s is valid: %s\n", path, describeRules(rules),
+			)
+			return err
+		},
+	}
+}

internal/cli/skiprules/command.go

@@ -0,0 +1,72 @@
+// Package skiprules provides the "skip-rules" command and its subcommands,
+// which check a skip rules file and apply it to the opportunities already
+// stored (see internal/skiprules).
+package skiprules
+
+import (
+	"errors"
+	"fmt"
+	"log/slog"
+
+	"github.com/spf13/cobra"
+
+	"git.avsdev.uk/Go/job-search/internal/cli"
+	"git.avsdev.uk/Go/job-search/internal/config"
+	"git.avsdev.uk/Go/job-search/internal/skiprules"
+)
+
+// NewCommand returns the "skip-rules" command, with "check" and "apply".
+func NewCommand(opts *cli.Options) *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   "skip-rules",
+		Short: "Check skip rules, or apply them to stored opportunities",
+		Long: `Skip rules mark new opportunities Skipped as the sources fetch
+them. They are read from sources.skip_rules_file, and read again when it
+changes.
+
+"check" reports whether a rules file is valid, and "apply" marks the stored
+opportunities that match Skipped. Both read sources.skip_rules_file unless
+given another file.`,
+		Args: cobra.NoArgs,
+		PersistentPreRun: func(cmd *cobra.Command, args []string) {
+			slog.Debug("command persistent pre-run", "command", "skip-rules")
+		},
+		PersistentPostRun: func(cmd *cobra.Command, args []string) {
+			slog.Debug("command persistent post-run", "command", "skip-rules")
+		},
+	}
+	cmd.AddCommand(commandCheck(opts))
+	cmd.AddCommand(commandApply(opts))
+
+	return cmd
+}
+
+// loadRules returns the rules in the file named by args, if given, or else
+// in sources.skip_rules_file, and the file's path.
+func loadRules(
+	conf *config.JobSearchConfig, args []string,
+) (*skiprules.Rules, string, error) {
+	path := conf.Sources.SkipRulesFile
+	if len(args) > 0 {
+		path = args[0]
+	}
+	if path == "" {
+		return nil, "", errors.New(
+			"no rules file given, and sources.skip_rules_file is not set",
+		)
+	}
+
+	rules, err := skiprules.Load(path)
+	if err != nil {
+		return nil, "", err
+	}
+
+	return rules, path, nil
+}
+
+// describeRules returns a summary of rules for output, e.g. "3 filters (2
+// active)".
+func describeRules(rules *skiprules.Rules) string {
+	filters, active := rules.Filters()
+	return fmt.Sprintf("%d filters (%d active)", filters, active)
+}

internal/cli/skiprules/skiprules_test.go

@@ -0,0 +1,175 @@
+package skiprules
+
+import (
+	"context"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/spf13/cobra"
+
+	"git.avsdev.uk/Go/job-search/internal/cli"
+	"git.avsdev.uk/Go/job-search/internal/cli/cmdtest"
+	"git.avsdev.uk/Go/job-search/internal/store"
+	"git.avsdev.uk/Go/job-search/internal/types"
+)
+
+// Run with: go test -tags sqlite_fts5 ./...
+//
+// These tests run the skip-rules commands as a user would, through a root
+// command like job-search's (see cmdtest).
+
+// conf is a config.yaml with a database and rules file in the test's
+// directory.
+const conf = "logging:\n  level: info\n" +
+	"database:\n  file: ./test.db\n" +
+	"sources:\n  skip_rules_file: ./rules.json\n"
+
+// rules skips buyer X's opportunities.
+const rules = `[{"comment": "buyer X", "active": true,
+	"rules": [{"rule_type": "buyer", "values": ["X"]}]}]`
+
+func newRoot() *cobra.Command {
+	opts := &cli.Options{}
+	return cmdtest.NewRoot(opts, NewCommand(opts))
+}
+
+// storeOpportunities creates test.db holding ocds-1 (buyer X), ocds-2 (buyer
+// Y) and ocds-3 (buyer X, already skipped and interested).
+func storeOpportunities(t *testing.T) {
+	t.Helper()
+	ctx := context.Background()
+	s, err := store.Open(ctx, "test.db", store.WithAutoMigrate())
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer s.Close()
+
+	buyers := []string{"X", "Y", "X"}
+	for i, buyer := range buyers {
+		uuid := "ocds-" + string(rune('1'+i))
+		_, err := s.SaveOpportunity(ctx, &types.Opportunity{
+			UUID:     uuid,
+			UUIDType: "ocid",
+			Title:    "Opportunity " + uuid,
+			BuyerID:  buyer,
+			Releases: []types.OpportunityRelease{
+				{
+					ReleaseID:   "r1",
+					ReleaseDate: time.Date(2026, 10, 1+i, 12, 0, 0, 0, time.UTC),
+					JSON:        "{}",
+				},
+			},
+		})
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+	err = s.SetMarks(ctx, types.OpportunityMarks{
+		UUID:       "ocds-3",
+		Skipped:    true,
+		Interested: true,
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
+// marks returns the stored marks of uuid.
+func marks(t *testing.T, uuid string) types.OpportunityMarks {
+	t.Helper()
+	s, err := store.Open(context.Background(), "test.db")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer s.Close()
+	opportunity, err := s.GetOpportunity(context.Background(), uuid)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return opportunity.Marks
+}
+
+func TestCheck(t *testing.T) {
+	cmdtest.Setup(t, conf)
+	cmdtest.WriteFile(t, "rules.json", rules)
+	cmdtest.WriteFile(t, "other.json", `[]`)
+	cmdtest.WriteFile(t, "bad.json", `[{"comment": "c"}]`)
+
+	out := cmdtest.MustRun(t, newRoot, "skip-rules", "check")
+	cmdtest.Contains(t, out, "./rules.json is valid: 1 filters (1 active)")
+
+	out = cmdtest.MustRun(t, newRoot, "skip-rules", "check", "other.json")
+	cmdtest.Contains(t, out, "other.json is valid: 0 filters (0 active)")
+
+	_, err := cmdtest.Run(t, newRoot, "", "skip-rules", "check", "bad.json")
+	cmdtest.ErrContains(t, err, `bad.json: filter 1 ("c"): skiprules: active`)
+
+	_, err = cmdtest.Run(t, newRoot, "", "skip-rules", "check", "none.json")
+	cmdtest.ErrContains(t, err, "none.json")
+}
+
+func TestCheckNoFile(t *testing.T) {
+	cmdtest.Setup(t, "logging:\n  level: info\n")
+	_, err := cmdtest.Run(t, newRoot, "", "skip-rules", "check")
+	cmdtest.ErrContains(t, err, "sources.skip_rules_file is not set")
+}
+
+func TestApplyDryRun(t *testing.T) {
+	cmdtest.Setup(t, conf)
+	cmdtest.WriteFile(t, "rules.json", rules)
+	storeOpportunities(t)
+
+	out := cmdtest.MustRun(t, newRoot, "skip-rules", "apply", "--dry-run")
+	cmdtest.Contains(
+		t,
+		out,
+		"ocds-1  buyer X  Opportunity ocds-1",
+		"1 opportunities would be skipped (dry run)",
+	)
+	if strings.Contains(out, "ocds-2") || strings.Contains(out, "ocds-3") {
+		t.Errorf("listed a non-match or an already skipped one:\n%s", out)
+	}
+	if marks(t, "ocds-1").Skipped {
+		t.Error("a dry run skipped ocds-1")
+	}
+}
+
+// apply sets only Skipped, on the matches not yet skipped, and leaves the
+// other marks alone.
+func TestApply(t *testing.T) {
+	cmdtest.Setup(t, conf)
+	cmdtest.WriteFile(t, "rules.json", rules)
+	storeOpportunities(t)
+
+	out := cmdtest.MustRun(t, newRoot, "skip-rules", "apply")
+	cmdtest.Contains(t, out, "ocds-1  buyer X", "Skipped 1 opportunities")
+
+	if !marks(t, "ocds-1").Skipped {
+		t.Error("ocds-1 wasn't skipped")
+	}
+	if marks(t, "ocds-2").Skipped {
+		t.Error("ocds-2 (buyer Y) was skipped")
+	}
+	if got := marks(t, "ocds-3"); !got.Skipped || !got.Interested {
+		t.Errorf("ocds-3's marks changed: %+v", got)
+	}
+
+	out = cmdtest.MustRun(t, newRoot, "skip-rules", "apply")
+	cmdtest.Contains(t, out, "Skipped 0 opportunities")
+}
+
+// A rules file given on the command line is used instead of the
+// configured one.
+func TestApplyGivenFile(t *testing.T) {
+	cmdtest.Setup(t, conf)
+	cmdtest.WriteFile(t, "rules.json", rules)
+	cmdtest.WriteFile(t, "y.json", strings.ReplaceAll(rules, `"X"`, `"Y"`))
+	storeOpportunities(t)
+
+	out := cmdtest.MustRun(t, newRoot, "skip-rules", "apply", "y.json")
+	cmdtest.Contains(t, out, "ocds-2", "Skipped 1 opportunities")
+	if !marks(t, "ocds-2").Skipped || marks(t, "ocds-1").Skipped {
+		t.Error("y.json's rules weren't the ones applied")
+	}
+}

internal/cli/skiprules_test.go

@@ -0,0 +1,43 @@
+package cli
+
+import (
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"git.avsdev.uk/Go/job-search/internal/config"
+)
+
+// SkipRulesOptions gives no option without a file, one with a valid file,
+// and refuses a missing or invalid file, naming the setting.
+func TestSkipRulesOptions(t *testing.T) {
+	dir := t.TempDir()
+	valid := filepath.Join(dir, "valid.json")
+	invalid := filepath.Join(dir, "invalid.json")
+	os.WriteFile(valid, []byte(`[]`), 0o600)
+	os.WriteFile(invalid, []byte(`[{}]`), 0o600)
+
+	tests := []struct {
+		file    string
+		options int
+		err     string
+	}{
+		{"", 0, ""},
+		{valid, 1, ""},
+		{filepath.Join(dir, "missing.json"), 0, "sources.skip_rules_file: "},
+		{invalid, 0, "sources.skip_rules_file: skip rules "},
+	}
+	for _, tt := range tests {
+		conf := &config.JobSearchConfig{}
+		conf.Sources.SkipRulesFile = tt.file
+		options, err := SkipRulesOptions(conf)
+		if len(options) != tt.options {
+			t.Errorf("%q: %d options; want %d", tt.file, len(options), tt.options)
+		}
+		if (err == nil) != (tt.err == "") ||
+			err != nil && !strings.Contains(err.Error(), tt.err) {
+			t.Errorf("%q: err %v; want %q", tt.file, err, tt.err)
+		}
+	}
+}