Added skip rules, which mark new opportunities Skipped when they're fetched
sources.skip_rules_file names a JSON file of filters, each with a
comment, an active flag, a logic mode ("or" by default, or "and") and
rules by buyer, title words or CPV codes. The fetcher applies them
before saving, so only new opportunities are marked; the marks of
stored ones are never touched. CPV rules match when the opportunity's
codes are all in the list, or any of them with exclusive false, and
by prefix with exact false.
The file is read again when its time or size changes: an invalid edit
is logged and the last good rules kept, and a deleted file means no
rules until it's back. fetcher service and serve refuse to start if it
is define but missing or invalid.
job-search skip-rules check validates a rules file, and skip-rules
apply marks the stored opportunities that match Skipped, or only lists
them with --dry-run.
This commit is contained in:
bwilliams committed
1 parent
2e6c21f9dd
commit
0543762330
21 files changed
+1766
-8
No files matched your search
@@ -0,0 +1,175 @@
|
||||
package skiprules
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
|
||||
"git.avsdev.uk/Go/job-search/internal/cli"
|
||||
"git.avsdev.uk/Go/job-search/internal/cli/cmdtest"
|
||||
"git.avsdev.uk/Go/job-search/internal/store"
|
||||
"git.avsdev.uk/Go/job-search/internal/types"
|
||||
)
|
||||
|
||||
// Run with: go test -tags sqlite_fts5 ./...
|
||||
//
|
||||
// These tests run the skip-rules commands as a user would, through a root
|
||||
// command like job-search's (see cmdtest).
|
||||
|
||||
// conf is a config.yaml with a database and rules file in the test's
|
||||
// directory.
|
||||
const conf = "logging:\n level: info\n" +
|
||||
"database:\n file: ./test.db\n" +
|
||||
"sources:\n skip_rules_file: ./rules.json\n"
|
||||
|
||||
// rules skips buyer X's opportunities.
|
||||
const rules = `[{"comment": "buyer X", "active": true,
|
||||
"rules": [{"rule_type": "buyer", "values": ["X"]}]}]`
|
||||
|
||||
func newRoot() *cobra.Command {
|
||||
opts := &cli.Options{}
|
||||
return cmdtest.NewRoot(opts, NewCommand(opts))
|
||||
}
|
||||
|
||||
// storeOpportunities creates test.db holding ocds-1 (buyer X), ocds-2 (buyer
|
||||
// Y) and ocds-3 (buyer X, already skipped and interested).
|
||||
func storeOpportunities(t *testing.T) {
|
||||
t.Helper()
|
||||
ctx := context.Background()
|
||||
s, err := store.Open(ctx, "test.db", store.WithAutoMigrate())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer s.Close()
|
||||
|
||||
buyers := []string{"X", "Y", "X"}
|
||||
for i, buyer := range buyers {
|
||||
uuid := "ocds-" + string(rune('1'+i))
|
||||
_, err := s.SaveOpportunity(ctx, &types.Opportunity{
|
||||
UUID: uuid,
|
||||
UUIDType: "ocid",
|
||||
Title: "Opportunity " + uuid,
|
||||
BuyerID: buyer,
|
||||
Releases: []types.OpportunityRelease{
|
||||
{
|
||||
ReleaseID: "r1",
|
||||
ReleaseDate: time.Date(2026, 10, 1+i, 12, 0, 0, 0, time.UTC),
|
||||
JSON: "{}",
|
||||
},
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
err = s.SetMarks(ctx, types.OpportunityMarks{
|
||||
UUID: "ocds-3",
|
||||
Skipped: true,
|
||||
Interested: true,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// marks returns the stored marks of uuid.
|
||||
func marks(t *testing.T, uuid string) types.OpportunityMarks {
|
||||
t.Helper()
|
||||
s, err := store.Open(context.Background(), "test.db")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer s.Close()
|
||||
opportunity, err := s.GetOpportunity(context.Background(), uuid)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return opportunity.Marks
|
||||
}
|
||||
|
||||
func TestCheck(t *testing.T) {
|
||||
cmdtest.Setup(t, conf)
|
||||
cmdtest.WriteFile(t, "rules.json", rules)
|
||||
cmdtest.WriteFile(t, "other.json", `[]`)
|
||||
cmdtest.WriteFile(t, "bad.json", `[{"comment": "c"}]`)
|
||||
|
||||
out := cmdtest.MustRun(t, newRoot, "skip-rules", "check")
|
||||
cmdtest.Contains(t, out, "./rules.json is valid: 1 filters (1 active)")
|
||||
|
||||
out = cmdtest.MustRun(t, newRoot, "skip-rules", "check", "other.json")
|
||||
cmdtest.Contains(t, out, "other.json is valid: 0 filters (0 active)")
|
||||
|
||||
_, err := cmdtest.Run(t, newRoot, "", "skip-rules", "check", "bad.json")
|
||||
cmdtest.ErrContains(t, err, `bad.json: filter 1 ("c"): skiprules: active`)
|
||||
|
||||
_, err = cmdtest.Run(t, newRoot, "", "skip-rules", "check", "none.json")
|
||||
cmdtest.ErrContains(t, err, "none.json")
|
||||
}
|
||||
|
||||
func TestCheckNoFile(t *testing.T) {
|
||||
cmdtest.Setup(t, "logging:\n level: info\n")
|
||||
_, err := cmdtest.Run(t, newRoot, "", "skip-rules", "check")
|
||||
cmdtest.ErrContains(t, err, "sources.skip_rules_file is not set")
|
||||
}
|
||||
|
||||
func TestApplyDryRun(t *testing.T) {
|
||||
cmdtest.Setup(t, conf)
|
||||
cmdtest.WriteFile(t, "rules.json", rules)
|
||||
storeOpportunities(t)
|
||||
|
||||
out := cmdtest.MustRun(t, newRoot, "skip-rules", "apply", "--dry-run")
|
||||
cmdtest.Contains(
|
||||
t,
|
||||
out,
|
||||
"ocds-1 buyer X Opportunity ocds-1",
|
||||
"1 opportunities would be skipped (dry run)",
|
||||
)
|
||||
if strings.Contains(out, "ocds-2") || strings.Contains(out, "ocds-3") {
|
||||
t.Errorf("listed a non-match or an already skipped one:\n%s", out)
|
||||
}
|
||||
if marks(t, "ocds-1").Skipped {
|
||||
t.Error("a dry run skipped ocds-1")
|
||||
}
|
||||
}
|
||||
|
||||
// apply sets only Skipped, on the matches not yet skipped, and leaves the
|
||||
// other marks alone.
|
||||
func TestApply(t *testing.T) {
|
||||
cmdtest.Setup(t, conf)
|
||||
cmdtest.WriteFile(t, "rules.json", rules)
|
||||
storeOpportunities(t)
|
||||
|
||||
out := cmdtest.MustRun(t, newRoot, "skip-rules", "apply")
|
||||
cmdtest.Contains(t, out, "ocds-1 buyer X", "Skipped 1 opportunities")
|
||||
|
||||
if !marks(t, "ocds-1").Skipped {
|
||||
t.Error("ocds-1 wasn't skipped")
|
||||
}
|
||||
if marks(t, "ocds-2").Skipped {
|
||||
t.Error("ocds-2 (buyer Y) was skipped")
|
||||
}
|
||||
if got := marks(t, "ocds-3"); !got.Skipped || !got.Interested {
|
||||
t.Errorf("ocds-3's marks changed: %+v", got)
|
||||
}
|
||||
|
||||
out = cmdtest.MustRun(t, newRoot, "skip-rules", "apply")
|
||||
cmdtest.Contains(t, out, "Skipped 0 opportunities")
|
||||
}
|
||||
|
||||
// A rules file given on the command line is used instead of the
|
||||
// configured one.
|
||||
func TestApplyGivenFile(t *testing.T) {
|
||||
cmdtest.Setup(t, conf)
|
||||
cmdtest.WriteFile(t, "rules.json", rules)
|
||||
cmdtest.WriteFile(t, "y.json", strings.ReplaceAll(rules, `"X"`, `"Y"`))
|
||||
storeOpportunities(t)
|
||||
|
||||
out := cmdtest.MustRun(t, newRoot, "skip-rules", "apply", "y.json")
|
||||
cmdtest.Contains(t, out, "ocds-2", "Skipped 1 opportunities")
|
||||
if !marks(t, "ocds-2").Skipped || marks(t, "ocds-1").Skipped {
|
||||
t.Error("y.json's rules weren't the ones applied")
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user