Naming, layout and comments: descriptive full-word names, one item per line in wrapped calls and literals, and doc comments on exported names
Short and single-letter names in application code become full words (tests keep theirs). Wrapped calls, signatures and composite literals put one item per line, and one-line function bodies are expanded. Exported names and groups of constants get doc comments, and comments refer to the renamed identifiers.
This commit is contained in:
1 parent
043e307663
commit
465358609a
78 files changed
+2374
-1402
No files matched your search
@@ -48,7 +48,9 @@ func writeFile(t *testing.T, path, contents string) {
|
||||
// root command has, after parsing args.
|
||||
func newCommand(t *testing.T, opts *Options, args ...string) *cobra.Command {
|
||||
t.Helper()
|
||||
cmd := &cobra.Command{Use: "test"}
|
||||
cmd := &cobra.Command{
|
||||
Use: "test",
|
||||
}
|
||||
opts.Bind(cmd)
|
||||
if err := cmd.ParseFlags(args); err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -203,7 +205,9 @@ func confirmWithInput(t *testing.T, input string) bool {
|
||||
|
||||
oldIn, oldOut := os.Stdin, os.Stdout
|
||||
os.Stdin, os.Stdout = stdin, stdout
|
||||
defer func() { os.Stdin, os.Stdout = oldIn, oldOut }()
|
||||
defer func() {
|
||||
os.Stdin, os.Stdout = oldIn, oldOut
|
||||
}()
|
||||
|
||||
return Confirm("Proceed?")
|
||||
}
|
||||
@@ -63,8 +63,12 @@ func TestConfigShow(t *testing.T) {
|
||||
cmdtest.Setup(t, appConfig)
|
||||
|
||||
out := mustRun(t, "config", "show")
|
||||
cmdtest.Contains(t, out,
|
||||
"logging.level = info <string>", "sources.sleeper.interval = 2m0s <string>")
|
||||
cmdtest.Contains(
|
||||
t,
|
||||
out,
|
||||
"logging.level = info <string>",
|
||||
"sources.sleeper.interval = 2m0s <string>",
|
||||
)
|
||||
// Keys are listed in order, the same every time.
|
||||
if again := mustRun(t, "config", "show"); again != out {
|
||||
t.Errorf("config show output changed between runs:\n%s\n---\n%s", out, again)
|
||||
@@ -158,8 +162,11 @@ func TestConfigSetUnset(t *testing.T) {
|
||||
t.Errorf("config.yaml settings = %v; want %v", got, want)
|
||||
}
|
||||
// Back to the default.
|
||||
cmdtest.Contains(t, mustRun(t, "config", "get", "sources.sleeper.interval"),
|
||||
"10m0s <string>")
|
||||
cmdtest.Contains(
|
||||
t,
|
||||
mustRun(t, "config", "get", "sources.sleeper.interval"),
|
||||
"10m0s <string>",
|
||||
)
|
||||
}
|
||||
|
||||
func TestConfigSetWithoutConfigFile(t *testing.T) {
|
||||
|
||||
@@ -32,12 +32,12 @@ func commandGet(opts *cli.Options) *cobra.Command {
|
||||
return fmt.Errorf("no configuration loaded (%w)", err)
|
||||
}
|
||||
|
||||
v, err := conf.ValueOf(args[0])
|
||||
value, err := conf.ValueOf(args[0])
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
t, _ := conf.TypeOf(args[0])
|
||||
fmt.Println(v, "<"+t.String()+">")
|
||||
kind, _ := conf.TypeOf(args[0])
|
||||
fmt.Println(value, "<"+kind.String()+">")
|
||||
|
||||
return nil
|
||||
},
|
||||
|
||||
@@ -74,7 +74,8 @@ func commandInit() *cobra.Command {
|
||||
fmt.Print(
|
||||
"writing default configuration to file: " +
|
||||
viper.ConfigFileUsed() +
|
||||
"\n")
|
||||
"\n",
|
||||
)
|
||||
|
||||
if err := viper.WriteConfig(); err != nil {
|
||||
return err
|
||||
|
||||
@@ -8,14 +8,14 @@ import (
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Confirm asks the user the yes/no question s on stdin and reports whether
|
||||
// Confirm asks the user the yes/no question prompt on stdin and reports whether
|
||||
// they answered yes. "y" and "yes" confirm, "n" and "no" refuse, in any case;
|
||||
// anything else asks again. It exits the program if stdin can't be read.
|
||||
func Confirm(s string) bool {
|
||||
func Confirm(prompt string) bool {
|
||||
reader := bufio.NewReader(os.Stdin)
|
||||
|
||||
for {
|
||||
fmt.Printf("%s [y/n]: ", s)
|
||||
fmt.Printf("%s [y/n]: ", prompt)
|
||||
|
||||
response, err := reader.ReadString('\n')
|
||||
if err != nil {
|
||||
|
||||
+10
-10
@@ -29,23 +29,23 @@ var stageTags = map[string][]string{
|
||||
|
||||
// filter keeps the opportunities whose latest release is at stage. Releases
|
||||
// are oldest first, so the last one is the procedure's current stage.
|
||||
func filter(opps []*types.Opportunity, stage string) []*types.Opportunity {
|
||||
var resp []*types.Opportunity
|
||||
func filter(opportunities []*types.Opportunity, stage string) []*types.Opportunity {
|
||||
var matching []*types.Opportunity
|
||||
|
||||
for _, opp := range opps {
|
||||
if len(opp.Releases) == 0 {
|
||||
for _, opportunity := range opportunities {
|
||||
if len(opportunity.Releases) == 0 {
|
||||
continue
|
||||
}
|
||||
latest := opp.Releases[len(opp.Releases)-1]
|
||||
latest := opportunity.Releases[len(opportunity.Releases)-1]
|
||||
for _, tag := range latest.Tags {
|
||||
if slices.Contains(stageTags[stage], tag.Tag) {
|
||||
resp = append(resp, opp)
|
||||
matching = append(matching, opportunity)
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return resp
|
||||
return matching
|
||||
}
|
||||
|
||||
func commandOnce(opts *cli.Options) *cobra.Command {
|
||||
@@ -71,12 +71,12 @@ func commandOnce(opts *cli.Options) *cobra.Command {
|
||||
return sinceErr
|
||||
}
|
||||
|
||||
ft := findatender.New(conf, conf.Sources.FindATender.APIRoot)
|
||||
defer ft.Shutdown()
|
||||
findATender := findatender.New(conf, conf.Sources.FindATender.APIRoot)
|
||||
defer findATender.Shutdown()
|
||||
|
||||
ignoreRateLimit, _ := cmd.Flags().GetBool("ignore-rate-limit")
|
||||
|
||||
results, err := ft.FetchAll(since, !ignoreRateLimit)
|
||||
results, err := findATender.FetchAll(since, !ignoreRateLimit)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -12,7 +12,9 @@ import (
|
||||
|
||||
// opp makes an opportunity whose releases, oldest first, have these tags.
|
||||
func opp(uuid string, tags ...string) *types.Opportunity {
|
||||
o := &types.Opportunity{UUID: uuid}
|
||||
o := &types.Opportunity{
|
||||
UUID: uuid,
|
||||
}
|
||||
for _, tag := range tags {
|
||||
o.Releases = append(o.Releases, types.OpportunityRelease{
|
||||
Tags: []types.OpportunityReleaseTag{{Tag: tag}},
|
||||
|
||||
@@ -59,12 +59,12 @@ func openMigrator(
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
m, err := store.NewMigrator(db)
|
||||
migrator, err := store.NewMigrator(db)
|
||||
if err != nil {
|
||||
return nil, nil, errors.Join(err, db.Close())
|
||||
}
|
||||
|
||||
return m, db, nil
|
||||
return migrator, db, nil
|
||||
}
|
||||
|
||||
// closeDB closes db when deferred, adding any error to *err.
|
||||
@@ -80,61 +80,72 @@ type errWriter struct {
|
||||
}
|
||||
|
||||
// printf writes like fmt.Fprintf, unless an earlier write failed.
|
||||
func (e *errWriter) printf(format string, a ...any) {
|
||||
func (e *errWriter) printf(format string, args ...any) {
|
||||
if e.err == nil {
|
||||
_, e.err = fmt.Fprintf(e.w, format, a...)
|
||||
_, e.err = fmt.Fprintf(e.w, format, args...)
|
||||
}
|
||||
}
|
||||
|
||||
// printResults lists migrations that ran, e.g.
|
||||
//
|
||||
// applied 00003_create_release_tags_table.sql (2ms)
|
||||
func printResults(w io.Writer, results []*goose.MigrationResult) error {
|
||||
ew := &errWriter{w: w}
|
||||
for _, r := range results {
|
||||
func printResults(out io.Writer, results []*goose.MigrationResult) error {
|
||||
output := &errWriter{
|
||||
w: out,
|
||||
}
|
||||
for _, result := range results {
|
||||
verb := "applied"
|
||||
if r.Direction == "down" {
|
||||
if result.Direction == "down" {
|
||||
verb = "reverted"
|
||||
}
|
||||
ew.printf(" %-9s %s (%s)\n",
|
||||
verb, filepath.Base(r.Source.Path), r.Duration.Round(time.Millisecond))
|
||||
output.printf(
|
||||
" %-9s %s (%s)\n",
|
||||
verb,
|
||||
filepath.Base(result.Source.Path),
|
||||
result.Duration.Round(time.Millisecond),
|
||||
)
|
||||
}
|
||||
return ew.err
|
||||
return output.err
|
||||
}
|
||||
|
||||
// failure prints what ran before a failed migration and describes the
|
||||
// failure. Each migration runs in a transaction, so the failed one has been
|
||||
// rolled back and the database is at the last version that succeeded.
|
||||
func failure(w io.Writer, err error) error {
|
||||
var pe *goose.PartialError
|
||||
if !errors.As(err, &pe) {
|
||||
func failure(out io.Writer, err error) error {
|
||||
var partialErr *goose.PartialError
|
||||
if !errors.As(err, &partialErr) {
|
||||
return err
|
||||
}
|
||||
|
||||
printErr := printResults(w, pe.Applied)
|
||||
printErr := printResults(out, partialErr.Applied)
|
||||
|
||||
name := "unknown migration"
|
||||
if pe.Failed != nil && pe.Failed.Source != nil {
|
||||
name = filepath.Base(pe.Failed.Source.Path)
|
||||
if partialErr.Failed != nil && partialErr.Failed.Source != nil {
|
||||
name = filepath.Base(partialErr.Failed.Source.Path)
|
||||
}
|
||||
|
||||
return errors.Join(fmt.Errorf(
|
||||
"migration %s failed and was rolled back; the database is at the last "+
|
||||
"version that succeeded.\nMigration error: %w",
|
||||
name, pe.Err), printErr)
|
||||
return errors.Join(
|
||||
fmt.Errorf(
|
||||
"migration %s failed and was rolled back; the database is at the last "+
|
||||
"version that succeeded.\nMigration error: %w",
|
||||
name,
|
||||
partialErr.Err,
|
||||
),
|
||||
printErr,
|
||||
)
|
||||
}
|
||||
|
||||
// printVersion reports where the database ended up.
|
||||
func printVersion(ctx context.Context, w io.Writer, m *goose.Provider) error {
|
||||
v, err := m.GetDBVersion(ctx)
|
||||
func printVersion(ctx context.Context, out io.Writer, migrator *goose.Provider) error {
|
||||
dbVersion, err := migrator.GetDBVersion(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if v == 0 {
|
||||
_, err = fmt.Fprintln(w, "Database cleared")
|
||||
if dbVersion == 0 {
|
||||
_, err = fmt.Fprintln(out, "Database cleared")
|
||||
} else {
|
||||
_, err = fmt.Fprintf(w, "Database migrated to version %d\n", v)
|
||||
_, err = fmt.Fprintf(out, "Database migrated to version %d\n", dbVersion)
|
||||
}
|
||||
return err
|
||||
}
|
||||
@@ -16,7 +16,9 @@ import (
|
||||
|
||||
func result(path, direction string) *goose.MigrationResult {
|
||||
return &goose.MigrationResult{
|
||||
Source: &goose.Source{Path: "migrations/" + path},
|
||||
Source: &goose.Source{
|
||||
Path: "migrations/" + path,
|
||||
},
|
||||
Direction: direction,
|
||||
Duration: 1500 * time.Microsecond,
|
||||
}
|
||||
@@ -45,7 +47,9 @@ func TestFailurePartial(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestFailureWithoutFailedSource(t *testing.T) {
|
||||
err := failure(&bytes.Buffer{}, &goose.PartialError{Err: errors.New("boom")})
|
||||
err := failure(&bytes.Buffer{}, &goose.PartialError{
|
||||
Err: errors.New("boom"),
|
||||
})
|
||||
if err == nil || !strings.Contains(err.Error(), "unknown migration") {
|
||||
t.Errorf("err = %v, want unknown migration", err)
|
||||
}
|
||||
|
||||
@@ -32,13 +32,13 @@ confirmed by providing -yes.`,
|
||||
yes, _ := cmd.Flags().GetBool("yes")
|
||||
steps, _ := cmd.Flags().GetUint("steps")
|
||||
|
||||
m, db, err := openMigrator(cmd, opts, false, false)
|
||||
migrator, db, err := openMigrator(cmd, opts, false, false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer closeDB(db, &err)
|
||||
|
||||
results, err := store.StepDown(cmd.Context(), m, int(steps), yes)
|
||||
results, err := store.StepDown(cmd.Context(), migrator, int(steps), yes)
|
||||
|
||||
if len(results) == 0 && err == nil {
|
||||
fmt.Println("No changes made to database (empty)")
|
||||
@@ -58,7 +58,7 @@ confirmed by providing -yes.`,
|
||||
return failure(cmd.OutOrStdout(), err)
|
||||
}
|
||||
|
||||
return printVersion(cmd.Context(), cmd.OutOrStdout(), m)
|
||||
return printVersion(cmd.Context(), cmd.OutOrStdout(), migrator)
|
||||
},
|
||||
}
|
||||
cmd.Flags().Uint(
|
||||
|
||||
@@ -55,7 +55,9 @@ func TestMigrateUpCreateThenStatus(t *testing.T) {
|
||||
latest := latestVersion(t)
|
||||
|
||||
out := mustRun(t, "migrate", "up", "--create")
|
||||
cmdtest.Contains(t, out,
|
||||
cmdtest.Contains(
|
||||
t,
|
||||
out,
|
||||
"applied 00001_create_opportunities_table.sql",
|
||||
fmt.Sprintf("Database migrated to version %d", latest),
|
||||
)
|
||||
@@ -64,7 +66,9 @@ func TestMigrateUpCreateThenStatus(t *testing.T) {
|
||||
}
|
||||
|
||||
out = mustRun(t, "migrate", "status")
|
||||
cmdtest.Contains(t, out,
|
||||
cmdtest.Contains(
|
||||
t,
|
||||
out,
|
||||
fmt.Sprintf("Database version %d of %d (up-to-date)", latest, latest),
|
||||
"00001_create_opportunities_table.sql",
|
||||
)
|
||||
@@ -87,8 +91,11 @@ func TestMigrateUpAsksBeforeCreating(t *testing.T) {
|
||||
|
||||
out, err := run(t, "n\n", "migrate", "up")
|
||||
cmdtest.ErrContains(t, err, "no database file")
|
||||
cmdtest.Contains(t, out,
|
||||
"Database file './test.db' does not exist. Would you like to create it?")
|
||||
cmdtest.Contains(
|
||||
t,
|
||||
out,
|
||||
"Database file './test.db' does not exist. Would you like to create it?",
|
||||
)
|
||||
if cmdtest.Exists("test.db") {
|
||||
t.Error("test.db was created after answering no")
|
||||
}
|
||||
@@ -110,8 +117,11 @@ func TestMigrateUpBySteps(t *testing.T) {
|
||||
cmdtest.Contains(t, out, "Database migrated to version 2")
|
||||
|
||||
out = mustRun(t, "migrate", "status")
|
||||
cmdtest.Contains(t, out,
|
||||
fmt.Sprintf("Database version 2 of %d (%d pending)", latest, latest-2))
|
||||
cmdtest.Contains(
|
||||
t,
|
||||
out,
|
||||
fmt.Sprintf("Database version 2 of %d (%d pending)", latest, latest-2),
|
||||
)
|
||||
}
|
||||
|
||||
func TestMigrateDown(t *testing.T) {
|
||||
@@ -130,8 +140,11 @@ func TestMigrateDown(t *testing.T) {
|
||||
_, err := run(t, "", args...)
|
||||
cmdtest.ErrContains(t, err, "pass the --yes flag")
|
||||
}
|
||||
cmdtest.Contains(t, mustRun(t, "migrate", "status"),
|
||||
fmt.Sprintf("Database version %d of %d", latest-1, latest))
|
||||
cmdtest.Contains(
|
||||
t,
|
||||
mustRun(t, "migrate", "status"),
|
||||
fmt.Sprintf("Database version %d of %d", latest-1, latest),
|
||||
)
|
||||
|
||||
out = mustRun(t, "migrate", "down", "--steps", "0", "--yes")
|
||||
cmdtest.Contains(t, out, "Database cleared")
|
||||
|
||||
@@ -22,51 +22,63 @@ import (
|
||||
// 1 applied 2026-10-07 10:12:01 00001_create_opportunities_table.sql
|
||||
// ...
|
||||
// 5 pending 00005_create_opportunity_classifications_table.sql
|
||||
func printStatus(ctx context.Context, w io.Writer, m *goose.Provider) error {
|
||||
statuses, err := m.Status(ctx)
|
||||
func printStatus(ctx context.Context, out io.Writer, migrator *goose.Provider) error {
|
||||
statuses, err := migrator.Status(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
current, err := m.GetDBVersion(ctx)
|
||||
current, err := migrator.GetDBVersion(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
latest := store.LatestVersion(m)
|
||||
latest := store.LatestVersion(migrator)
|
||||
|
||||
pending := 0
|
||||
for _, s := range statuses {
|
||||
if s.State == goose.StatePending {
|
||||
for _, status := range statuses {
|
||||
if status.State == goose.StatePending {
|
||||
pending++
|
||||
}
|
||||
}
|
||||
|
||||
ew := &errWriter{w: w}
|
||||
output := &errWriter{
|
||||
w: out,
|
||||
}
|
||||
switch {
|
||||
case current > latest:
|
||||
ew.printf("Database version %d is newer than this build (latest %d)\n",
|
||||
current, latest)
|
||||
output.printf(
|
||||
"Database version %d is newer than this build (latest %d)\n",
|
||||
current,
|
||||
latest,
|
||||
)
|
||||
case pending == 0:
|
||||
ew.printf("Database version %d of %d (up-to-date)\n", current, latest)
|
||||
output.printf("Database version %d of %d (up-to-date)\n", current, latest)
|
||||
default:
|
||||
ew.printf("Database version %d of %d (%d pending)\n", current, latest, pending)
|
||||
output.printf("Database version %d of %d (%d pending)\n", current, latest, pending)
|
||||
}
|
||||
if ew.err != nil {
|
||||
return ew.err
|
||||
if output.err != nil {
|
||||
return output.err
|
||||
}
|
||||
|
||||
tw := tabwriter.NewWriter(w, 0, 0, 2, ' ', 0)
|
||||
ew = &errWriter{w: tw}
|
||||
for _, s := range statuses {
|
||||
applied := ""
|
||||
if s.State == goose.StateApplied {
|
||||
applied = s.AppliedAt.Local().Format("2006-01-02 15:04:05")
|
||||
}
|
||||
ew.printf(" %d\t%s\t%s\t%s\n",
|
||||
s.Source.Version, s.State, applied, filepath.Base(s.Source.Path))
|
||||
table := tabwriter.NewWriter(out, 0, 0, 2, ' ', 0)
|
||||
output = &errWriter{
|
||||
w: table,
|
||||
}
|
||||
return errors.Join(ew.err, tw.Flush())
|
||||
for _, status := range statuses {
|
||||
applied := ""
|
||||
if status.State == goose.StateApplied {
|
||||
applied = status.AppliedAt.Local().Format("2006-01-02 15:04:05")
|
||||
}
|
||||
output.printf(
|
||||
" %d\t%s\t%s\t%s\n",
|
||||
status.Source.Version,
|
||||
status.State,
|
||||
applied,
|
||||
filepath.Base(status.Source.Path),
|
||||
)
|
||||
}
|
||||
return errors.Join(output.err, table.Flush())
|
||||
}
|
||||
|
||||
func commandStatus(opts *cli.Options) *cobra.Command {
|
||||
@@ -83,13 +95,13 @@ func commandStatus(opts *cli.Options) *cobra.Command {
|
||||
RunE: func(cmd *cobra.Command, args []string) (err error) {
|
||||
slog.Debug("command run", "command", "migrate.status")
|
||||
|
||||
m, db, err := openMigrator(cmd, opts, false, false)
|
||||
migrator, db, err := openMigrator(cmd, opts, false, false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer closeDB(db, &err)
|
||||
|
||||
return printStatus(cmd.Context(), cmd.OutOrStdout(), m)
|
||||
return printStatus(cmd.Context(), cmd.OutOrStdout(), migrator)
|
||||
},
|
||||
}
|
||||
}
|
||||
@@ -16,13 +16,13 @@ import (
|
||||
func runStepUp(
|
||||
cmd *cobra.Command, opts *cli.Options, steps int, create bool,
|
||||
) (err error) {
|
||||
m, db, err := openMigrator(cmd, opts, true, create)
|
||||
migrator, db, err := openMigrator(cmd, opts, true, create)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer closeDB(db, &err)
|
||||
|
||||
results, err := store.StepUp(cmd.Context(), m, int(steps))
|
||||
results, err := store.StepUp(cmd.Context(), migrator, int(steps))
|
||||
|
||||
if len(results) == 0 && err == nil {
|
||||
_, err := fmt.Fprintln(cmd.OutOrStdout(), "No changes made to database (up-to-date)")
|
||||
@@ -37,7 +37,7 @@ func runStepUp(
|
||||
return failure(cmd.OutOrStdout(), err)
|
||||
}
|
||||
|
||||
return printVersion(cmd.Context(), cmd.OutOrStdout(), m)
|
||||
return printVersion(cmd.Context(), cmd.OutOrStdout(), migrator)
|
||||
}
|
||||
|
||||
func commandUp(opts *cli.Options) *cobra.Command {
|
||||
|
||||
@@ -20,10 +20,10 @@ type Options struct {
|
||||
|
||||
// Bind registers the shared flags as persistent flags on the root command cmd.
|
||||
func (o *Options) Bind(cmd *cobra.Command) {
|
||||
pf := cmd.PersistentFlags()
|
||||
pf.StringVarP(&o.configFile, "config", "c", "", "configuration file")
|
||||
pf.BoolVarP(&o.debug, "debug", "d", false, "enable debug messages")
|
||||
pf.BoolVarP(
|
||||
flags := cmd.PersistentFlags()
|
||||
flags.StringVarP(&o.configFile, "config", "c", "", "configuration file")
|
||||
flags.BoolVarP(&o.debug, "debug", "d", false, "enable debug messages")
|
||||
flags.BoolVarP(
|
||||
&o.quiet, "quiet", "q", false, "don't report info or warning messages",
|
||||
)
|
||||
}
|
||||
|
||||
@@ -17,6 +17,8 @@ import (
|
||||
"git.avsdev.uk/Go/job-search/internal/websocket/tcpserver"
|
||||
)
|
||||
|
||||
// NewCommand returns the wsserve command, which runs a plain TCP server for
|
||||
// websocket hub clients (see tcpserver).
|
||||
func NewCommand(opts *cli.Options) *cobra.Command {
|
||||
cmd := &cobra.Command{
|
||||
Use: "wsserve",
|
||||
@@ -43,7 +45,7 @@ func NewCommand(opts *cli.Options) *cobra.Command {
|
||||
}
|
||||
i, err := strconv.Atoi(listenParts[1])
|
||||
if err != nil {
|
||||
return fmt.Errorf("Failed to parse --listen port: %w", err)
|
||||
return fmt.Errorf("failed to parse --listen port: %w", err)
|
||||
}
|
||||
hostOverride = listenParts[0]
|
||||
portOverride = i
|
||||
@@ -75,9 +77,9 @@ func NewCommand(opts *cli.Options) *cobra.Command {
|
||||
}()
|
||||
|
||||
hub := websocket.NewHub(conf.Server.WebSocket)
|
||||
svr := tcpserver.New(&conf.Server, hub)
|
||||
server := tcpserver.New(&conf.Server, hub)
|
||||
|
||||
return svr.Run(ctx)
|
||||
return server.Run(ctx)
|
||||
},
|
||||
}
|
||||
cmd.Flags().String("listen", "", "listen address (default from config, 127.0.0.1:9393)")
|
||||
|
||||
@@ -50,7 +50,8 @@ func (c *JSCLogging) validate(key string, value any) error {
|
||||
case "debug", "info", "warn", "error":
|
||||
default:
|
||||
return errors.New(
|
||||
"invalid log level. Must be one of: debug, info, warn, error")
|
||||
"invalid log level. Must be one of: debug, info, warn, error",
|
||||
)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
|
||||
@@ -25,8 +25,12 @@ func captureLogs(t *testing.T, level slog.Level) *bytes.Buffer {
|
||||
t.Helper()
|
||||
var buf bytes.Buffer
|
||||
def := slog.Default()
|
||||
slog.SetDefault(slog.New(slog.NewTextHandler(&buf, &slog.HandlerOptions{Level: level})))
|
||||
t.Cleanup(func() { slog.SetDefault(def) })
|
||||
slog.SetDefault(slog.New(slog.NewTextHandler(&buf, &slog.HandlerOptions{
|
||||
Level: level,
|
||||
})))
|
||||
t.Cleanup(func() {
|
||||
slog.SetDefault(def)
|
||||
})
|
||||
return &buf
|
||||
}
|
||||
|
||||
@@ -62,7 +66,10 @@ func TestDefaults(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestLoggingSection(t *testing.T) {
|
||||
c := JSCLogging{Level: "info", File: "/tmp/x.log"}
|
||||
c := JSCLogging{
|
||||
Level: "info",
|
||||
File: "/tmp/x.log",
|
||||
}
|
||||
|
||||
if got, want := c.Keys(), []string{"level", "file"}; !reflect.DeepEqual(got, want) {
|
||||
t.Errorf("Keys = %v; want %v", got, want)
|
||||
@@ -142,7 +149,9 @@ func TestConfigLevelToLogLevel(t *testing.T) {
|
||||
if got := ConfigLevelToLogLevel(in); got != want {
|
||||
t.Errorf("ConfigLevelToLogLevel(%q) = %v; want %v", in, got, want)
|
||||
}
|
||||
c := JSCLogging{Level: in}
|
||||
c := JSCLogging{
|
||||
Level: in,
|
||||
}
|
||||
if got := c.LogLevel(); got != want {
|
||||
t.Errorf("LogLevel() with %q = %v; want %v", in, got, want)
|
||||
}
|
||||
@@ -150,7 +159,10 @@ func TestConfigLevelToLogLevel(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestSourceSection(t *testing.T) {
|
||||
c := JSCSourcesSource{Enabled: true, Interval: time.Minute}
|
||||
c := JSCSourcesSource{
|
||||
Enabled: true,
|
||||
Interval: time.Minute,
|
||||
}
|
||||
|
||||
if got, want := c.Keys(), []string{"enabled", "interval"}; !reflect.DeepEqual(got, want) {
|
||||
t.Errorf("Keys = %v; want %v", got, want)
|
||||
@@ -458,7 +470,9 @@ func TestLoadFromViperInvalidValues(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestDatabaseSection(t *testing.T) {
|
||||
c := JSCDatabase{File: "./jobsearch.db"}
|
||||
c := JSCDatabase{
|
||||
File: "./jobsearch.db",
|
||||
}
|
||||
|
||||
if got, want := c.Keys(), []string{"file"}; !reflect.DeepEqual(got, want) {
|
||||
t.Errorf("Keys = %v; want %v", got, want)
|
||||
|
||||
+19
-19
@@ -62,8 +62,8 @@ func (c *JobSearchConfig) SetValue(key string, value any) error {
|
||||
// Unknown keys are skipped; invalid values are logged and ignored.
|
||||
func LoadFromViper(conf *JobSearchConfig) {
|
||||
keys := viper.AllKeys()
|
||||
for idx := range keys {
|
||||
key := keys[idx]
|
||||
for i := range keys {
|
||||
key := keys[i]
|
||||
keyType, _ := conf.TypeOf(key)
|
||||
if keyType == reflect.Invalid {
|
||||
continue
|
||||
@@ -94,15 +94,15 @@ func viperValue(key string, kind reflect.Kind) (value any, ok bool) {
|
||||
// SetValue to split on commas; GetStringSlice would split it on
|
||||
// spaces. A list of objects (a list of sections) is passed on as it
|
||||
// is, as GetStringSlice would flatten it.
|
||||
switch v := viper.Get(key).(type) {
|
||||
switch value := viper.Get(key).(type) {
|
||||
case string:
|
||||
return v, true
|
||||
return value, true
|
||||
case []map[string]any: // as LoadToViper sets it
|
||||
return v, true
|
||||
return value, true
|
||||
case []any:
|
||||
if len(v) > 0 {
|
||||
if _, ok := v[0].(map[string]any); ok {
|
||||
return v, true
|
||||
if len(value) > 0 {
|
||||
if _, ok := value[0].(map[string]any); ok {
|
||||
return value, true
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -121,26 +121,26 @@ func LoadToViper(conf JobSearchConfig) {
|
||||
|
||||
// String formats the configuration one "key = value <type>" line per key, with
|
||||
// dotted keys, sorted by key.
|
||||
func (conf JobSearchConfig) String() string {
|
||||
var sb strings.Builder
|
||||
flat := conf.Flatten()
|
||||
for _, k := range slices.Sorted(maps.Keys(flat)) {
|
||||
typ, _ := conf.TypeOf(k)
|
||||
fmt.Fprint(&sb, k, " = ", flat[k], " <", typ, ">\n")
|
||||
func (c JobSearchConfig) String() string {
|
||||
var builder strings.Builder
|
||||
flat := c.Flatten()
|
||||
for _, key := range slices.Sorted(maps.Keys(flat)) {
|
||||
kind, _ := c.TypeOf(key)
|
||||
fmt.Fprint(&builder, key, " = ", flat[key], " <", kind, ">\n")
|
||||
}
|
||||
return sb.String()
|
||||
return builder.String()
|
||||
}
|
||||
|
||||
// JSON formats the configuration as indented JSON.
|
||||
func (conf JobSearchConfig) JSON() string {
|
||||
confMap := conf.Map()
|
||||
func (c JobSearchConfig) JSON() string {
|
||||
confMap := c.Map()
|
||||
jbytes, _ := json.MarshalIndent(confMap, "", " ")
|
||||
return string(jbytes)
|
||||
}
|
||||
|
||||
// YAML formats the configuration as YAML.
|
||||
func (conf JobSearchConfig) YAML() string {
|
||||
confMap := conf.Map()
|
||||
func (c JobSearchConfig) YAML() string {
|
||||
confMap := c.Map()
|
||||
ybytes, _ := yaml.Marshal(confMap)
|
||||
return string(ybytes)
|
||||
}
|
||||
@@ -70,7 +70,14 @@ func countSections(uses map[reflect.Type]int, section reflect.Type) {
|
||||
}
|
||||
}
|
||||
|
||||
func checkSections(t *testing.T, conf *JobSearchConfig, all map[string]any, uses map[reflect.Type]int, prefix string, section reflect.Value) {
|
||||
func checkSections(
|
||||
t *testing.T,
|
||||
conf *JobSearchConfig,
|
||||
all map[string]any,
|
||||
uses map[reflect.Type]int,
|
||||
prefix string,
|
||||
section reflect.Value,
|
||||
) {
|
||||
for _, f := range fieldsOf(section.Type()) {
|
||||
key := joinKey(prefix, f.key)
|
||||
if isSectionList(f.typ) {
|
||||
@@ -93,8 +100,12 @@ func checkSections(t *testing.T, conf *JobSearchConfig, all map[string]any, uses
|
||||
}
|
||||
for k := range keys {
|
||||
if _, ok := all[joinKey(keyPrefix, k)]; !ok {
|
||||
t.Errorf("%s.Flatten() key %q does not match its cfg tag %q",
|
||||
name, k, key)
|
||||
t.Errorf(
|
||||
"%s.Flatten() key %q does not match its cfg tag %q",
|
||||
name,
|
||||
k,
|
||||
key,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -116,6 +127,7 @@ func checkSections(t *testing.T, conf *JobSearchConfig, all map[string]any, uses
|
||||
// checkSectionList checks the items of the list of sections at key implement
|
||||
// Section and that their methods agree with the list's value in all.
|
||||
func checkSectionList(t *testing.T, all map[string]any, key string, list reflect.Value) {
|
||||
t.Helper()
|
||||
name := list.Type().Elem().Name()
|
||||
if !reflect.PointerTo(list.Type().Elem()).Implements(sectionType) {
|
||||
t.Errorf("*%s (items of %s) does not implement Section", name, key)
|
||||
@@ -145,7 +157,13 @@ func checkSectionList(t *testing.T, all map[string]any, key string, list reflect
|
||||
// checkSectionMethods checks section's Map, TypeOf, ValueOf and SetValue
|
||||
// against want, its keys and values, and kinds, their kinds.
|
||||
// Setting each key to its own value must succeed and leave section unchanged.
|
||||
func checkSectionMethods(t *testing.T, name string, section reflect.Value, want map[string]any, kinds map[string]reflect.Kind) {
|
||||
func checkSectionMethods(
|
||||
t *testing.T,
|
||||
name string,
|
||||
section reflect.Value,
|
||||
want map[string]any,
|
||||
kinds map[string]reflect.Kind,
|
||||
) {
|
||||
s := section.Addr().Interface().(Section)
|
||||
if got := flattenMap("", s.Map()); !reflect.DeepEqual(got, want) {
|
||||
t.Errorf("%s.Map() =\n%v\nwant\n%v", name, got, want)
|
||||
@@ -184,14 +202,18 @@ func flattenMap(prefix string, m map[string]any) map[string]any {
|
||||
// checkEmbedded checks that structs embedded in a section, such as
|
||||
// JSCSourcesSource, implement Section too.
|
||||
func checkEmbedded(t *testing.T, section reflect.Type) {
|
||||
t.Helper()
|
||||
for i := 0; i < section.NumField(); i++ {
|
||||
sf := section.Field(i)
|
||||
if !sf.Anonymous || sf.Type.Kind() != reflect.Struct {
|
||||
continue
|
||||
}
|
||||
if !reflect.PointerTo(sf.Type).Implements(sectionType) {
|
||||
t.Errorf("*%s (embedded in %s) does not implement Section",
|
||||
sf.Type.Name(), section.Name())
|
||||
t.Errorf(
|
||||
"*%s (embedded in %s) does not implement Section",
|
||||
sf.Type.Name(),
|
||||
section.Name(),
|
||||
)
|
||||
}
|
||||
checkEmbedded(t, sf.Type)
|
||||
}
|
||||
|
||||
@@ -22,7 +22,9 @@ func initDir(t *testing.T, env []string, config string) string {
|
||||
// godotenv sets variables without cleaning up after the test.
|
||||
for _, kv := range env {
|
||||
key, _, _ := strings.Cut(kv, "=")
|
||||
t.Cleanup(func() { os.Unsetenv(key) })
|
||||
t.Cleanup(func() {
|
||||
os.Unsetenv(key)
|
||||
})
|
||||
}
|
||||
writeFile(t, filepath.Join(dir, ".env"), strings.Join(env, "\n")+"\n")
|
||||
}
|
||||
@@ -108,7 +110,11 @@ func TestInitKeepsCommandLineLevel(t *testing.T) {
|
||||
initDir(t, []string{}, "logging:\n level: info\n")
|
||||
logs := captureLogs(t, slog.LevelDebug)
|
||||
|
||||
conf := JobSearchConfig{Logging: JSCLogging{Level: level}}
|
||||
conf := JobSearchConfig{
|
||||
Logging: JSCLogging{
|
||||
Level: level,
|
||||
},
|
||||
}
|
||||
if err := Init("", &conf); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
@@ -21,8 +21,8 @@ func StoreValue(
|
||||
|
||||
// storeValue is StoreValue for any section, so key types that
|
||||
// JobSearchConfig doesn't have yet can be tested.
|
||||
func storeValue(s Section, key string, value string) (any, reflect.Kind, error) {
|
||||
confType, err := s.TypeOf(key)
|
||||
func storeValue(section Section, key string, value string) (any, reflect.Kind, error) {
|
||||
confType, err := section.TypeOf(key)
|
||||
if err != nil {
|
||||
return nil, confType, err
|
||||
}
|
||||
@@ -38,10 +38,10 @@ func storeValue(s Section, key string, value string) (any, reflect.Kind, error)
|
||||
// SetValue parses the string into the field's own type (e.g. int8,
|
||||
// float32 or time.Duration) and rejects values the configuration itself
|
||||
// would not accept, e.g. an unknown log level or an unparseable duration.
|
||||
if err := s.SetValue(key, value); err != nil {
|
||||
if err := section.SetValue(key, value); err != nil {
|
||||
return nil, confType, err
|
||||
}
|
||||
confValue, err := s.ValueOf(key)
|
||||
confValue, err := section.ValueOf(key)
|
||||
if err != nil {
|
||||
return nil, confType, err
|
||||
}
|
||||
@@ -93,9 +93,9 @@ func PurgeValue(conf *JobSearchConfig, key string) error {
|
||||
}
|
||||
out := viper.New()
|
||||
out.SetConfigFile(file.ConfigFileUsed())
|
||||
for _, k := range file.AllKeys() {
|
||||
if k != key {
|
||||
out.Set(k, file.Get(k))
|
||||
for _, fileKey := range file.AllKeys() {
|
||||
if fileKey != key {
|
||||
out.Set(fileKey, file.Get(fileKey))
|
||||
}
|
||||
}
|
||||
return out.WriteConfig()
|
||||
|
||||
@@ -11,18 +11,18 @@ import (
|
||||
)
|
||||
|
||||
// PrintTerminal prints the configuration to stdout as String formats it.
|
||||
func (conf JobSearchConfig) PrintTerminal() {
|
||||
fmt.Print(conf.String() + "\n")
|
||||
func (c JobSearchConfig) PrintTerminal() {
|
||||
fmt.Print(c.String() + "\n")
|
||||
}
|
||||
|
||||
// PrintJson prints the configuration to stdout as JSON.
|
||||
func (conf JobSearchConfig) PrintJson() {
|
||||
fmt.Print(conf.JSON() + "\n")
|
||||
func (c JobSearchConfig) PrintJson() {
|
||||
fmt.Print(c.JSON() + "\n")
|
||||
}
|
||||
|
||||
// PrintYaml prints the configuration to stdout as YAML.
|
||||
func (conf JobSearchConfig) PrintYaml() {
|
||||
fmt.Print(conf.YAML() + "\n")
|
||||
func (c JobSearchConfig) PrintYaml() {
|
||||
fmt.Print(c.YAML() + "\n")
|
||||
}
|
||||
|
||||
// PrintLog logs each configuration key and value, sorted by key, at atLevel:
|
||||
@@ -55,4 +55,4 @@ func (conf JobSearchConfig) PrintLog(atLevel slog.Level) {
|
||||
for _, key := range keys {
|
||||
logger.Log(ctx, atLevel, "config value", "key", key, "value", flat[key])
|
||||
}
|
||||
}
|
||||
}
|
||||
+197
-179
@@ -26,7 +26,8 @@ import (
|
||||
//
|
||||
// time.Duration fields report reflect.String from TypeOf and accept either a
|
||||
// time.Duration or a duration string such as "90s" in SetValue.
|
||||
// util.ByteSize fields work the same way, with size strings such as "32KiB".
|
||||
// util.ByteSize fields work the same way, with size strings such as "32KiB"
|
||||
// (or "-1" for no limit).
|
||||
//
|
||||
// []string fields report reflect.Slice and accept either a []string or a
|
||||
// comma-separated string such as "a, b" in SetValue (and so in default tags).
|
||||
@@ -42,16 +43,25 @@ import (
|
||||
//
|
||||
// A section can check values before they are stored by implementing
|
||||
// validator on its pointer type.
|
||||
// The struct tags section.go reads.
|
||||
const (
|
||||
keyTag = "cfg"
|
||||
// keyTag holds a field's configuration key.
|
||||
keyTag = "cfg"
|
||||
// defaultTag holds a field's default value.
|
||||
defaultTag = "default"
|
||||
)
|
||||
|
||||
// The unknown key error and the field types section.go handles specially.
|
||||
var (
|
||||
errUnknownKey = errors.New("unknown configuration key")
|
||||
durationType = reflect.TypeOf(time.Duration(0))
|
||||
byteSizeType = reflect.TypeOf(util.ByteSize(0))
|
||||
timeType = reflect.TypeOf(time.Time{})
|
||||
// errUnknownKey is returned for a key that names no field.
|
||||
errUnknownKey = errors.New("unknown configuration key")
|
||||
// durationType is time.Duration, read and given out as a string.
|
||||
durationType = reflect.TypeOf(time.Duration(0))
|
||||
// byteSizeType is util.ByteSize, read and given out as a string.
|
||||
byteSizeType = reflect.TypeOf(util.ByteSize(0))
|
||||
// timeType is time.Time, a struct that is a value, not a section.
|
||||
timeType = reflect.TypeOf(time.Time{})
|
||||
// stringSliceType is []string, set from a comma-separated list.
|
||||
stringSliceType = reflect.TypeOf([]string{})
|
||||
)
|
||||
|
||||
@@ -100,67 +110,67 @@ type field struct {
|
||||
hasDefault bool
|
||||
}
|
||||
|
||||
// isSectionList reports whether t is a list of sections, []S for a section
|
||||
// struct S.
|
||||
func isSectionList(t reflect.Type) bool {
|
||||
return t.Kind() == reflect.Slice && t.Elem().Kind() == reflect.Struct &&
|
||||
t.Elem() != timeType
|
||||
// isSectionList reports whether listType is a list of sections, []S for a
|
||||
// section struct S.
|
||||
func isSectionList(listType reflect.Type) bool {
|
||||
return listType.Kind() == reflect.Slice && listType.Elem().Kind() == reflect.Struct &&
|
||||
listType.Elem() != timeType
|
||||
}
|
||||
|
||||
// fieldsOf lists the configuration fields of a section type in declaration
|
||||
// order, with the fields of embedded structs promoted into it.
|
||||
func fieldsOf(t reflect.Type) []field {
|
||||
func fieldsOf(structType reflect.Type) []field {
|
||||
var fields []field
|
||||
for i := 0; i < t.NumField(); i++ {
|
||||
sf := t.Field(i)
|
||||
if sf.Anonymous && sf.Type.Kind() == reflect.Struct {
|
||||
for _, f := range fieldsOf(sf.Type) {
|
||||
f.index = append([]int{i}, f.index...)
|
||||
fields = append(fields, f)
|
||||
for i := 0; i < structType.NumField(); i++ {
|
||||
structField := structType.Field(i)
|
||||
if structField.Anonymous && structField.Type.Kind() == reflect.Struct {
|
||||
for _, embeddedField := range fieldsOf(structField.Type) {
|
||||
embeddedField.index = append([]int{i}, embeddedField.index...)
|
||||
fields = append(fields, embeddedField)
|
||||
}
|
||||
continue
|
||||
}
|
||||
key, ok := sf.Tag.Lookup(keyTag)
|
||||
if !ok || key == "" || key == "-" || !sf.IsExported() {
|
||||
key, ok := structField.Tag.Lookup(keyTag)
|
||||
if !ok || key == "" || key == "-" || !structField.IsExported() {
|
||||
continue
|
||||
}
|
||||
def, hasDefault := sf.Tag.Lookup(defaultTag)
|
||||
defaultValue, hasDefault := structField.Tag.Lookup(defaultTag)
|
||||
fields = append(fields, field{
|
||||
key: key,
|
||||
index: []int{i},
|
||||
typ: sf.Type,
|
||||
section: sf.Type.Kind() == reflect.Struct && sf.Type != timeType,
|
||||
def: def,
|
||||
typ: structField.Type,
|
||||
section: structField.Type.Kind() == reflect.Struct && structField.Type != timeType,
|
||||
def: defaultValue,
|
||||
hasDefault: hasDefault,
|
||||
})
|
||||
}
|
||||
return fields
|
||||
}
|
||||
|
||||
// structValue dereferences v down to the section struct it holds.
|
||||
func structValue(v any) reflect.Value {
|
||||
rv := reflect.ValueOf(v)
|
||||
for rv.Kind() == reflect.Pointer {
|
||||
rv = rv.Elem()
|
||||
// structValue dereferences value down to the section struct it holds.
|
||||
func structValue(value any) reflect.Value {
|
||||
reflected := reflect.ValueOf(value)
|
||||
for reflected.Kind() == reflect.Pointer {
|
||||
reflected = reflected.Elem()
|
||||
}
|
||||
if rv.Kind() != reflect.Struct {
|
||||
panic(fmt.Sprintf("config: %T is not a configuration section", v))
|
||||
if reflected.Kind() != reflect.Struct {
|
||||
panic(fmt.Sprintf("config: %T is not a configuration section", value))
|
||||
}
|
||||
return rv
|
||||
return reflected
|
||||
}
|
||||
|
||||
// lookup resolves a dotted key to the leaf field and the section holding it.
|
||||
func lookup(section reflect.Value, key string) (reflect.Value, field, error) {
|
||||
head, rest, nested := strings.Cut(key, ".")
|
||||
for _, f := range fieldsOf(section.Type()) {
|
||||
if f.key != head {
|
||||
for _, configField := range fieldsOf(section.Type()) {
|
||||
if configField.key != head {
|
||||
continue
|
||||
}
|
||||
switch {
|
||||
case f.section && nested:
|
||||
return lookup(section.FieldByIndex(f.index), rest)
|
||||
case !f.section && !nested:
|
||||
return section, f, nil
|
||||
case configField.section && nested:
|
||||
return lookup(section.FieldByIndex(configField.index), rest)
|
||||
case !configField.section && !nested:
|
||||
return section, configField, nil
|
||||
}
|
||||
// Key matched but the shape didn't: a section with no sub-key, or a
|
||||
// value with one.
|
||||
@@ -177,52 +187,52 @@ func joinKey(prefix, key string) string {
|
||||
}
|
||||
|
||||
// keysOf returns the keys directly in a section.
|
||||
func keysOf(v any) []string {
|
||||
fields := fieldsOf(structValue(v).Type())
|
||||
func keysOf(value any) []string {
|
||||
fields := fieldsOf(structValue(value).Type())
|
||||
keys := make([]string, 0, len(fields))
|
||||
for _, f := range fields {
|
||||
keys = append(keys, f.key)
|
||||
for _, configField := range fields {
|
||||
keys = append(keys, configField.key)
|
||||
}
|
||||
return keys
|
||||
}
|
||||
|
||||
// mapOf converts a section to a map, with nested sections as nested maps.
|
||||
func mapOf(v any) map[string]any {
|
||||
return mapOfValue(structValue(v))
|
||||
func mapOf(value any) map[string]any {
|
||||
return mapOfValue(structValue(value))
|
||||
}
|
||||
|
||||
// mapOfValue is mapOf for a section struct value.
|
||||
func mapOfValue(section reflect.Value) map[string]any {
|
||||
m := map[string]any{}
|
||||
for _, f := range fieldsOf(section.Type()) {
|
||||
fv := section.FieldByIndex(f.index)
|
||||
if f.section {
|
||||
m[f.key] = mapOfValue(fv)
|
||||
values := map[string]any{}
|
||||
for _, configField := range fieldsOf(section.Type()) {
|
||||
fieldValue := section.FieldByIndex(configField.index)
|
||||
if configField.section {
|
||||
values[configField.key] = mapOfValue(fieldValue)
|
||||
} else {
|
||||
m[f.key] = exportValue(fv)
|
||||
values[configField.key] = exportValue(fieldValue)
|
||||
}
|
||||
}
|
||||
return m
|
||||
return values
|
||||
}
|
||||
|
||||
// flattenOf converts a section to a flat map of dotted keys, each prefixed
|
||||
// with prefix (if non-empty).
|
||||
func flattenOf(prefix string, v any) map[string]any {
|
||||
m := map[string]any{}
|
||||
flattenInto(m, prefix, structValue(v))
|
||||
return m
|
||||
func flattenOf(prefix string, value any) map[string]any {
|
||||
flat := map[string]any{}
|
||||
flattenInto(flat, prefix, structValue(value))
|
||||
return flat
|
||||
}
|
||||
|
||||
// flattenInto adds section's keys to m, dotted and prefixed with prefix (if
|
||||
// non-empty), descending into nested sections.
|
||||
func flattenInto(m map[string]any, prefix string, section reflect.Value) {
|
||||
for _, f := range fieldsOf(section.Type()) {
|
||||
fv := section.FieldByIndex(f.index)
|
||||
key := joinKey(prefix, f.key)
|
||||
if f.section {
|
||||
flattenInto(m, key, fv)
|
||||
func flattenInto(flat map[string]any, prefix string, section reflect.Value) {
|
||||
for _, configField := range fieldsOf(section.Type()) {
|
||||
fieldValue := section.FieldByIndex(configField.index)
|
||||
key := joinKey(prefix, configField.key)
|
||||
if configField.section {
|
||||
flattenInto(flat, key, fieldValue)
|
||||
} else {
|
||||
m[key] = exportValue(fv)
|
||||
flat[key] = exportValue(fieldValue)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -232,33 +242,33 @@ func flattenInto(m map[string]any, prefix string, section reflect.Value) {
|
||||
// so values agree with kindOf and can be passed back to SetValue; []string is
|
||||
// copied; a list of sections becomes a []map[string]any (nil if empty);
|
||||
// everything else is unchanged.
|
||||
func exportValue(fv reflect.Value) any {
|
||||
switch fv.Type() {
|
||||
func exportValue(fieldValue reflect.Value) any {
|
||||
switch fieldValue.Type() {
|
||||
case durationType:
|
||||
return time.Duration(fv.Int()).String()
|
||||
return time.Duration(fieldValue.Int()).String()
|
||||
case byteSizeType:
|
||||
return util.ByteSize(fv.Int()).String()
|
||||
return util.ByteSize(fieldValue.Int()).String()
|
||||
}
|
||||
if isSectionList(fv.Type()) {
|
||||
if isSectionList(fieldValue.Type()) {
|
||||
var items []map[string]any
|
||||
for i := range fv.Len() {
|
||||
items = append(items, mapOfValue(fv.Index(i)))
|
||||
for i := range fieldValue.Len() {
|
||||
items = append(items, mapOfValue(fieldValue.Index(i)))
|
||||
}
|
||||
return items
|
||||
}
|
||||
v := fv.Interface()
|
||||
if s, ok := v.([]string); ok {
|
||||
value := fieldValue.Interface()
|
||||
if list, ok := value.([]string); ok {
|
||||
// A copy, so changing it doesn't change the configuration.
|
||||
return slices.Clone(s)
|
||||
return slices.Clone(list)
|
||||
}
|
||||
return v
|
||||
return value
|
||||
}
|
||||
|
||||
// splitList splits a comma-separated list, trimming spaces around each item
|
||||
// and dropping empty ones, so "" is an empty list.
|
||||
func splitList(s string) []string {
|
||||
func splitList(list string) []string {
|
||||
var items []string
|
||||
for item := range strings.SplitSeq(s, ",") {
|
||||
for item := range strings.SplitSeq(list, ",") {
|
||||
if item = strings.TrimSpace(item); item != "" {
|
||||
items = append(items, item)
|
||||
}
|
||||
@@ -268,115 +278,120 @@ func splitList(s string) []string {
|
||||
|
||||
// kindOf is the kind reported by TypeOf, i.e. how the value is read from
|
||||
// Viper. Durations and byte sizes are read as strings and parsed by SetValue.
|
||||
func kindOf(t reflect.Type) reflect.Kind {
|
||||
if t == durationType || t == byteSizeType {
|
||||
func kindOf(fieldType reflect.Type) reflect.Kind {
|
||||
if fieldType == durationType || fieldType == byteSizeType {
|
||||
return reflect.String
|
||||
}
|
||||
return t.Kind()
|
||||
return fieldType.Kind()
|
||||
}
|
||||
|
||||
// typeOf returns the kind of the dotted key in section v (see kindOf), or
|
||||
// errUnknownKey.
|
||||
func typeOf(v any, key string) (reflect.Kind, error) {
|
||||
_, f, err := lookup(structValue(v), key)
|
||||
// typeOf returns the kind of the dotted key in the section value (see
|
||||
// kindOf), or errUnknownKey.
|
||||
func typeOf(value any, key string) (reflect.Kind, error) {
|
||||
_, configField, err := lookup(structValue(value), key)
|
||||
if err != nil {
|
||||
return reflect.Invalid, err
|
||||
}
|
||||
return kindOf(f.typ), nil
|
||||
return kindOf(configField.typ), nil
|
||||
}
|
||||
|
||||
// valueOf returns the value of the dotted key in section v, or errUnknownKey.
|
||||
func valueOf(v any, key string) (any, error) {
|
||||
section, f, err := lookup(structValue(v), key)
|
||||
// valueOf returns the value of the dotted key in the section value, or
|
||||
// errUnknownKey.
|
||||
func valueOf(value any, key string) (any, error) {
|
||||
section, configField, err := lookup(structValue(value), key)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return exportValue(section.FieldByIndex(f.index)), nil
|
||||
return exportValue(section.FieldByIndex(configField.index)), nil
|
||||
}
|
||||
|
||||
// setValue sets a dotted key on the section that ptr points to.
|
||||
func setValue(ptr any, key string, value any) error {
|
||||
section, f, err := lookup(structValue(ptr), key)
|
||||
section, configField, err := lookup(structValue(ptr), key)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return setField(section, f, value)
|
||||
return setField(section, configField, value)
|
||||
}
|
||||
|
||||
// setField converts value to the field's type and, if the section has a
|
||||
// validator, checks it, before setting the field. On error the field is left
|
||||
// unchanged.
|
||||
func setField(section reflect.Value, f field, value any) error {
|
||||
newValue, err := convert(value, f.typ)
|
||||
func setField(section reflect.Value, configField field, value any) error {
|
||||
newValue, err := convert(value, configField.typ)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if v, ok := section.Addr().Interface().(validator); ok {
|
||||
if err := v.validate(f.key, newValue.Interface()); err != nil {
|
||||
if checker, ok := section.Addr().Interface().(validator); ok {
|
||||
if err := checker.validate(configField.key, newValue.Interface()); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
section.FieldByIndex(f.index).Set(newValue)
|
||||
section.FieldByIndex(configField.index).Set(newValue)
|
||||
return nil
|
||||
}
|
||||
|
||||
// convert turns value into a value of type t. Values already of the right
|
||||
// convert turns value into a value of type targetType. Values already of the right
|
||||
// type are used as-is, except that an empty []string becomes nil, as an empty
|
||||
// comma-separated list does (see splitList); strings are parsed. Lists of
|
||||
// sections are converted by convertSectionList.
|
||||
func convert(value any, t reflect.Type) (reflect.Value, error) {
|
||||
if isSectionList(t) {
|
||||
return convertSectionList(value, t)
|
||||
func convert(value any, targetType reflect.Type) (reflect.Value, error) {
|
||||
if isSectionList(targetType) {
|
||||
return convertSectionList(value, targetType)
|
||||
}
|
||||
if value != nil {
|
||||
rv := reflect.ValueOf(value)
|
||||
if rv.Type().AssignableTo(t) {
|
||||
if t == stringSliceType && rv.Len() == 0 {
|
||||
return reflect.Zero(t), nil
|
||||
reflected := reflect.ValueOf(value)
|
||||
if reflected.Type().AssignableTo(targetType) {
|
||||
if targetType == stringSliceType && reflected.Len() == 0 {
|
||||
return reflect.Zero(targetType), nil
|
||||
}
|
||||
return rv, nil
|
||||
return reflected, nil
|
||||
}
|
||||
if s, ok := value.(string); ok {
|
||||
return parseString(s, t)
|
||||
if text, ok := value.(string); ok {
|
||||
return parseString(text, targetType)
|
||||
}
|
||||
}
|
||||
return reflect.Value{}, fmt.Errorf("unsupported type %T", value)
|
||||
}
|
||||
|
||||
// convertSectionList turns value into a list of sections of type t: a value
|
||||
// of type t, a list of maps of keys to values, or a JSON array of objects.
|
||||
// convertSectionList turns value into a list of sections of type listType: a
|
||||
// value of type listType, a list of maps of keys to values, or a JSON array of objects.
|
||||
// nil and empty lists become a nil list.
|
||||
func convertSectionList(value any, t reflect.Type) (reflect.Value, error) {
|
||||
if s, ok := value.(string); ok {
|
||||
func convertSectionList(value any, listType reflect.Type) (reflect.Value, error) {
|
||||
if text, ok := value.(string); ok {
|
||||
var items []map[string]any
|
||||
if strings.TrimSpace(s) != "" {
|
||||
if err := json.Unmarshal([]byte(s), &items); err != nil {
|
||||
return reflect.Value{}, fmt.Errorf("unable to convert to a list of %s: %w", t.Elem(), err)
|
||||
if strings.TrimSpace(text) != "" {
|
||||
if err := json.Unmarshal([]byte(text), &items); err != nil {
|
||||
return reflect.Value{}, fmt.Errorf(
|
||||
"unable to convert to a list of %s: %w",
|
||||
listType.Elem(),
|
||||
err,
|
||||
)
|
||||
}
|
||||
}
|
||||
value = items
|
||||
}
|
||||
if value == nil {
|
||||
return reflect.Zero(t), nil
|
||||
return reflect.Zero(listType), nil
|
||||
}
|
||||
rv := reflect.ValueOf(value)
|
||||
if rv.Kind() != reflect.Slice {
|
||||
reflected := reflect.ValueOf(value)
|
||||
if reflected.Kind() != reflect.Slice {
|
||||
return reflect.Value{}, fmt.Errorf("unsupported type %T", value)
|
||||
}
|
||||
if rv.Len() == 0 {
|
||||
return reflect.Zero(t), nil
|
||||
if reflected.Len() == 0 {
|
||||
return reflect.Zero(listType), nil
|
||||
}
|
||||
if rv.Type().AssignableTo(t) {
|
||||
for i := range rv.Len() {
|
||||
if err := validateSection(rv.Index(i)); err != nil {
|
||||
if reflected.Type().AssignableTo(listType) {
|
||||
for i := range reflected.Len() {
|
||||
if err := validateSection(reflected.Index(i)); err != nil {
|
||||
return reflect.Value{}, fmt.Errorf("item %d: %w", i, err)
|
||||
}
|
||||
}
|
||||
return rv, nil
|
||||
return reflected, nil
|
||||
}
|
||||
list := reflect.MakeSlice(t, 0, rv.Len())
|
||||
for i := range rv.Len() {
|
||||
item, err := sectionFromMap(rv.Index(i).Interface(), t.Elem())
|
||||
list := reflect.MakeSlice(listType, 0, reflected.Len())
|
||||
for i := range reflected.Len() {
|
||||
item, err := sectionFromMap(reflected.Index(i).Interface(), listType.Elem())
|
||||
if err != nil {
|
||||
return reflect.Value{}, fmt.Errorf("item %d: %w", i, err)
|
||||
}
|
||||
@@ -385,31 +400,31 @@ func convertSectionList(value any, t reflect.Type) (reflect.Value, error) {
|
||||
return list, nil
|
||||
}
|
||||
|
||||
// sectionFromMap builds a section of type t from a map of its keys to values,
|
||||
// starting from its defaults. Values that are neither strings nor of the
|
||||
// field's type (e.g. numbers read from YAML) are formatted and parsed. The
|
||||
// finished section is validated as a whole (see validateSection), so a key
|
||||
// left out of the map can't skip its validation.
|
||||
func sectionFromMap(value any, t reflect.Type) (reflect.Value, error) {
|
||||
// sectionFromMap builds a section of type structType from value, a map of its
|
||||
// keys to values, starting from its defaults. Values that are neither strings
|
||||
// nor of the field's type (e.g. numbers read from YAML) are formatted and
|
||||
// parsed. The finished section is validated as a whole (see
|
||||
// validateSection), so a key left out of the map can't skip its validation.
|
||||
func sectionFromMap(value any, structType reflect.Type) (reflect.Value, error) {
|
||||
item, ok := value.(map[string]any)
|
||||
if !ok {
|
||||
return reflect.Value{}, fmt.Errorf("unsupported type %T", value)
|
||||
}
|
||||
section := reflect.New(t).Elem()
|
||||
section := reflect.New(structType).Elem()
|
||||
if err := applyDefaultsTo("", section); err != nil {
|
||||
return reflect.Value{}, err
|
||||
}
|
||||
for k, v := range item {
|
||||
s, f, err := lookup(section, k)
|
||||
for itemKey, itemValue := range item {
|
||||
holder, configField, err := lookup(section, itemKey)
|
||||
if err != nil {
|
||||
return reflect.Value{}, fmt.Errorf("%w %q", err, k)
|
||||
return reflect.Value{}, fmt.Errorf("%w %q", err, itemKey)
|
||||
}
|
||||
if _, isString := v.(string); v != nil && !isString &&
|
||||
!reflect.TypeOf(v).AssignableTo(f.typ) {
|
||||
v = fmt.Sprint(v)
|
||||
if _, isString := itemValue.(string); itemValue != nil && !isString &&
|
||||
!reflect.TypeOf(itemValue).AssignableTo(configField.typ) {
|
||||
itemValue = fmt.Sprint(itemValue)
|
||||
}
|
||||
if err := setField(s, f, v); err != nil {
|
||||
return reflect.Value{}, fmt.Errorf("%s: %w", k, err)
|
||||
if err := setField(holder, configField, itemValue); err != nil {
|
||||
return reflect.Value{}, fmt.Errorf("%s: %w", itemKey, err)
|
||||
}
|
||||
}
|
||||
if err := validateSection(section); err != nil {
|
||||
@@ -422,73 +437,76 @@ func sectionFromMap(value any, t reflect.Type) (reflect.Value, error) {
|
||||
// field directly in it, so values that were never set (defaults or zero
|
||||
// values) are checked too.
|
||||
func validateSection(section reflect.Value) error {
|
||||
v, ok := reflect.New(section.Type()).Interface().(validator)
|
||||
checker, ok := reflect.New(section.Type()).Interface().(validator)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
for _, f := range fieldsOf(section.Type()) {
|
||||
if f.section {
|
||||
for _, configField := range fieldsOf(section.Type()) {
|
||||
if configField.section {
|
||||
continue
|
||||
}
|
||||
if err := v.validate(f.key, section.FieldByIndex(f.index).Interface()); err != nil {
|
||||
return fmt.Errorf("%s: %w", f.key, err)
|
||||
if err := checker.validate(
|
||||
configField.key,
|
||||
section.FieldByIndex(configField.index).Interface(),
|
||||
); err != nil {
|
||||
return fmt.Errorf("%s: %w", configField.key, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// parseString parses s as a value of type t. It supports strings, bools,
|
||||
// parseString parses text as a value of type targetType. It supports strings, bools,
|
||||
// signed and unsigned integers, floats, time.Duration, ByteSize and []string
|
||||
// (from a comma-separated list, see splitList).
|
||||
func parseString(s string, t reflect.Type) (reflect.Value, error) {
|
||||
v := reflect.New(t).Elem()
|
||||
switch t {
|
||||
func parseString(text string, targetType reflect.Type) (reflect.Value, error) {
|
||||
parsed := reflect.New(targetType).Elem()
|
||||
switch targetType {
|
||||
case durationType:
|
||||
d, err := time.ParseDuration(s)
|
||||
duration, err := time.ParseDuration(text)
|
||||
if err != nil {
|
||||
return reflect.Value{}, fmt.Errorf("unable to convert to duration: %w", err)
|
||||
}
|
||||
v.SetInt(int64(d))
|
||||
return v, nil
|
||||
parsed.SetInt(int64(duration))
|
||||
return parsed, nil
|
||||
case byteSizeType:
|
||||
b, err := util.ParseByteSize(s)
|
||||
size, err := util.ParseByteSize(text)
|
||||
if err != nil {
|
||||
return reflect.Value{}, fmt.Errorf("unable to convert to byte size: %w", err)
|
||||
}
|
||||
v.SetInt(int64(b))
|
||||
return v, nil
|
||||
parsed.SetInt(int64(size))
|
||||
return parsed, nil
|
||||
case stringSliceType:
|
||||
v.Set(reflect.ValueOf(splitList(s)))
|
||||
return v, nil
|
||||
parsed.Set(reflect.ValueOf(splitList(text)))
|
||||
return parsed, nil
|
||||
}
|
||||
|
||||
var err error
|
||||
switch t.Kind() {
|
||||
switch targetType.Kind() {
|
||||
case reflect.String:
|
||||
v.SetString(s)
|
||||
parsed.SetString(text)
|
||||
case reflect.Bool:
|
||||
var b bool
|
||||
b, err = strconv.ParseBool(s)
|
||||
v.SetBool(b)
|
||||
var flag bool
|
||||
flag, err = strconv.ParseBool(text)
|
||||
parsed.SetBool(flag)
|
||||
case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
|
||||
var n int64
|
||||
n, err = strconv.ParseInt(s, 10, t.Bits())
|
||||
v.SetInt(n)
|
||||
var number int64
|
||||
number, err = strconv.ParseInt(text, 10, targetType.Bits())
|
||||
parsed.SetInt(number)
|
||||
case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
|
||||
var n uint64
|
||||
n, err = strconv.ParseUint(s, 10, t.Bits())
|
||||
v.SetUint(n)
|
||||
var number uint64
|
||||
number, err = strconv.ParseUint(text, 10, targetType.Bits())
|
||||
parsed.SetUint(number)
|
||||
case reflect.Float32, reflect.Float64:
|
||||
var n float64
|
||||
n, err = strconv.ParseFloat(s, t.Bits())
|
||||
v.SetFloat(n)
|
||||
var number float64
|
||||
number, err = strconv.ParseFloat(text, targetType.Bits())
|
||||
parsed.SetFloat(number)
|
||||
default:
|
||||
return reflect.Value{}, fmt.Errorf("unsupported type string for %s", t)
|
||||
return reflect.Value{}, fmt.Errorf("unsupported type string for %s", targetType)
|
||||
}
|
||||
if err != nil {
|
||||
return reflect.Value{}, fmt.Errorf("unable to convert %q to %s: %w", s, t, err)
|
||||
return reflect.Value{}, fmt.Errorf("unable to convert %q to %s: %w", text, targetType, err)
|
||||
}
|
||||
return v, nil
|
||||
return parsed, nil
|
||||
}
|
||||
|
||||
// applyDefaults sets every field that has a default tag, recursing into
|
||||
@@ -501,18 +519,18 @@ func applyDefaults(ptr any) error {
|
||||
// recursing into nested sections. prefix is the section's dotted key, used in
|
||||
// error messages.
|
||||
func applyDefaultsTo(prefix string, section reflect.Value) error {
|
||||
for _, f := range fieldsOf(section.Type()) {
|
||||
key := joinKey(prefix, f.key)
|
||||
if f.section {
|
||||
if err := applyDefaultsTo(key, section.FieldByIndex(f.index)); err != nil {
|
||||
for _, configField := range fieldsOf(section.Type()) {
|
||||
key := joinKey(prefix, configField.key)
|
||||
if configField.section {
|
||||
if err := applyDefaultsTo(key, section.FieldByIndex(configField.index)); err != nil {
|
||||
return err
|
||||
}
|
||||
continue
|
||||
}
|
||||
if !f.hasDefault {
|
||||
if !configField.hasDefault {
|
||||
continue
|
||||
}
|
||||
if err := setField(section, f, f.def); err != nil {
|
||||
if err := setField(section, configField, configField.def); err != nil {
|
||||
return fmt.Errorf("invalid default for %s: %w", key, err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -119,7 +119,9 @@ func TestMapOf(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestFlattenOfPrefix(t *testing.T) {
|
||||
m := flattenOf("p", testSection{Dur: 90 * time.Second})
|
||||
m := flattenOf("p", testSection{
|
||||
Dur: 90 * time.Second,
|
||||
})
|
||||
for _, k := range []string{"p.str", "p.on", "p.inner.name"} {
|
||||
if _, ok := m[k]; !ok {
|
||||
t.Errorf("flattenOf missing %q in %v", k, m)
|
||||
@@ -145,14 +147,18 @@ func TestApplyDefaults(t *testing.T) {
|
||||
t.Fatalf("applyDefaults: %v", err)
|
||||
}
|
||||
want := testSection{
|
||||
testEmbedded: testEmbedded{On: true},
|
||||
Str: "hello",
|
||||
Int: -42,
|
||||
Uint: 7,
|
||||
Float: 1.5,
|
||||
Dur: 90 * time.Second,
|
||||
Size: 32 << 10,
|
||||
Inner: testInner{Name: "inner"},
|
||||
testEmbedded: testEmbedded{
|
||||
On: true,
|
||||
},
|
||||
Str: "hello",
|
||||
Int: -42,
|
||||
Uint: 7,
|
||||
Float: 1.5,
|
||||
Dur: 90 * time.Second,
|
||||
Size: 32 << 10,
|
||||
Inner: testInner{
|
||||
Name: "inner",
|
||||
},
|
||||
}
|
||||
if !reflect.DeepEqual(s, want) {
|
||||
t.Errorf("applyDefaults = %+v; want %+v", s, want)
|
||||
@@ -247,6 +253,7 @@ func TestSetValue(t *testing.T) {
|
||||
{"size", "1 mb", "1MB"}, // byte sizes are given out as strings
|
||||
{"size", "1.5KiB", "1536B"},
|
||||
{"size", "4096", "4KiB"},
|
||||
{"size", "-1", "-1"}, // no limit
|
||||
{"slice", " a, b ,, c ", []string{"a", "b", "c"}},
|
||||
{"slice", "", []string(nil)},
|
||||
{"inner.name", "nested", "nested"},
|
||||
@@ -286,8 +293,14 @@ func TestSetValue(t *testing.T) {
|
||||
}
|
||||
got, err := valueOf(s, tt.key)
|
||||
if err != nil || !reflect.DeepEqual(got, tt.want) {
|
||||
t.Errorf("after setValue(%q, %#v): valueOf = %#v, %v; want %#v",
|
||||
tt.key, tt.value, got, err, tt.want)
|
||||
t.Errorf(
|
||||
"after setValue(%q, %#v): valueOf = %#v, %v; want %#v",
|
||||
tt.key,
|
||||
tt.value,
|
||||
got,
|
||||
err,
|
||||
tt.want,
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -322,8 +335,13 @@ func TestSetValueErrors(t *testing.T) {
|
||||
before := s
|
||||
err := setValue(&s, tt.key, tt.value)
|
||||
if err == nil || !strings.Contains(err.Error(), tt.want) {
|
||||
t.Errorf("setValue(%q, %#v) = %v; want error containing %q",
|
||||
tt.key, tt.value, err, tt.want)
|
||||
t.Errorf(
|
||||
"setValue(%q, %#v) = %v; want error containing %q",
|
||||
tt.key,
|
||||
tt.value,
|
||||
err,
|
||||
tt.want,
|
||||
)
|
||||
}
|
||||
if !reflect.DeepEqual(s, before) {
|
||||
t.Errorf("setValue(%q, %#v) modified the section on error", tt.key, tt.value)
|
||||
@@ -334,7 +352,9 @@ func TestSetValueErrors(t *testing.T) {
|
||||
// A []string is given out as a copy, so changing it leaves the section as it
|
||||
// was.
|
||||
func TestValueOfSliceIsACopy(t *testing.T) {
|
||||
s := testSection{Slice: []string{"a", "b"}}
|
||||
s := testSection{
|
||||
Slice: []string{"a", "b"},
|
||||
}
|
||||
got, err := valueOf(s, "slice")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
|
||||
@@ -12,6 +12,8 @@ import (
|
||||
"git.avsdev.uk/Go/job-search/internal/util"
|
||||
)
|
||||
|
||||
// ErrNoNewReleases is returned by a source's task when there was nothing new
|
||||
// to fetch; it wraps util.ErrWorkerNoWork.
|
||||
var ErrNoNewReleases = fmt.Errorf("%w: no new releases", util.ErrWorkerNoWork)
|
||||
|
||||
// Option configures Run.
|
||||
|
||||
@@ -77,7 +77,9 @@ func TestFetchFunctions(t *testing.T) {
|
||||
"releases found",
|
||||
http.StatusOK,
|
||||
onePackage,
|
||||
func(err error) bool { return err == nil },
|
||||
func(err error) bool {
|
||||
return err == nil
|
||||
},
|
||||
"nil",
|
||||
},
|
||||
{"no releases", http.StatusOK, emptyPackage, func(err error) bool {
|
||||
@@ -92,7 +94,10 @@ func TestFetchFunctions(t *testing.T) {
|
||||
t.Run(fname+"/"+tc.name, func(t *testing.T) {
|
||||
srv, hits := ftsServer(t, tc.status, tc.body)
|
||||
err := fetch(
|
||||
context.Background(), slog.Default(), confFor(srv.URL), time.Now().Add(-time.Hour),
|
||||
context.Background(),
|
||||
slog.Default(),
|
||||
confFor(srv.URL),
|
||||
time.Now().Add(-time.Hour),
|
||||
)
|
||||
if !tc.check(err) {
|
||||
t.Fatalf("err = %v, want %s", err, tc.wantErr)
|
||||
@@ -204,7 +209,9 @@ func TestRunLogsSourceErrors(t *testing.T) {
|
||||
var buf bytes.Buffer
|
||||
def := slog.Default()
|
||||
slog.SetDefault(slog.New(slog.NewTextHandler(&buf, nil)))
|
||||
t.Cleanup(func() { slog.SetDefault(def) })
|
||||
t.Cleanup(func() {
|
||||
slog.SetDefault(def)
|
||||
})
|
||||
|
||||
// A zero interval makes each worker fail straight away.
|
||||
conf := &config.JobSearchConfig{}
|
||||
@@ -253,7 +260,9 @@ func TestRunShutsDownOnWorkerError(t *testing.T) {
|
||||
conf.Sources.Sleeper.Enabled = true
|
||||
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- Run(context.Background(), conf, time.Now().Add(-time.Hour)) }()
|
||||
go func() {
|
||||
done <- Run(context.Background(), conf, time.Now().Add(-time.Hour))
|
||||
}()
|
||||
|
||||
select {
|
||||
case err := <-done:
|
||||
|
||||
@@ -15,9 +15,12 @@ import (
|
||||
// runFindAOpportunity polls Find a Tender every configured interval until ctx
|
||||
// is cancelled.
|
||||
func runFindAOpportunity(
|
||||
ctx context.Context, logger *slog.Logger, conf *config.JobSearchConfig, since time.Time,
|
||||
ctx context.Context,
|
||||
logger *slog.Logger,
|
||||
conf *config.JobSearchConfig,
|
||||
since time.Time,
|
||||
) error {
|
||||
w := util.Worker{
|
||||
worker := util.Worker{
|
||||
Logger: logger,
|
||||
Name: "FindAOpportunity",
|
||||
Interval: conf.Sources.FindATender.Interval,
|
||||
@@ -25,19 +28,26 @@ func runFindAOpportunity(
|
||||
return fetchFindAOpportunity(ctx, logger, conf, since)
|
||||
},
|
||||
}
|
||||
return w.Run(ctx, since)
|
||||
return worker.Run(ctx, since)
|
||||
}
|
||||
|
||||
// fetchFindAOpportunity fetches everything updated since since from Find a
|
||||
// Tender. Finding nothing returns ErrNoNewReleases, so the worker still moves
|
||||
// its checkpoint on.
|
||||
func fetchFindAOpportunity(
|
||||
ctx context.Context, logger *slog.Logger, conf *config.JobSearchConfig, since time.Time,
|
||||
ctx context.Context,
|
||||
logger *slog.Logger,
|
||||
conf *config.JobSearchConfig,
|
||||
since time.Time,
|
||||
) error {
|
||||
opportunities, err := util.AwaitTask(ctx, func() ([]*types.Opportunity, error) {
|
||||
ft := findatender.New(conf, conf.Sources.FindATender.APIRoot, findatender.WithLogger(logger))
|
||||
defer ft.Shutdown()
|
||||
return ft.With(ctx).FetchAll(since, true)
|
||||
findATender := findatender.New(
|
||||
conf,
|
||||
conf.Sources.FindATender.APIRoot,
|
||||
findatender.WithLogger(logger),
|
||||
)
|
||||
defer findATender.Shutdown()
|
||||
return findATender.With(ctx).FetchAll(since, true)
|
||||
})
|
||||
if errors.Is(err, findatender.ErrNoReleases) {
|
||||
return ErrNoNewReleases
|
||||
|
||||
@@ -15,9 +15,12 @@ import (
|
||||
// runFTSMock polls the mock Find a Tender server every configured interval
|
||||
// until ctx is cancelled.
|
||||
func runFTSMock(
|
||||
ctx context.Context, logger *slog.Logger, conf *config.JobSearchConfig, since time.Time,
|
||||
ctx context.Context,
|
||||
logger *slog.Logger,
|
||||
conf *config.JobSearchConfig,
|
||||
since time.Time,
|
||||
) error {
|
||||
w := util.Worker{
|
||||
worker := util.Worker{
|
||||
Logger: logger,
|
||||
Name: "FTSMock",
|
||||
Interval: conf.Sources.FTSMock.Interval,
|
||||
@@ -25,19 +28,26 @@ func runFTSMock(
|
||||
return fetchFTSMock(ctx, logger, conf, since)
|
||||
},
|
||||
}
|
||||
return w.Run(ctx, since)
|
||||
return worker.Run(ctx, since)
|
||||
}
|
||||
|
||||
// fetchFTSMock fetches everything updated since since from the mock Find a
|
||||
// Tender server. Finding nothing returns ErrNoNewReleases, so the worker still
|
||||
// moves its checkpoint on.
|
||||
func fetchFTSMock(
|
||||
ctx context.Context, logger *slog.Logger, conf *config.JobSearchConfig, since time.Time,
|
||||
ctx context.Context,
|
||||
logger *slog.Logger,
|
||||
conf *config.JobSearchConfig,
|
||||
since time.Time,
|
||||
) error {
|
||||
opportunities, err := util.AwaitTask(ctx, func() ([]*types.Opportunity, error) {
|
||||
ft := findatender.New(conf, conf.Sources.FTSMock.APIRoot, findatender.WithLogger(logger))
|
||||
defer ft.Shutdown()
|
||||
return ft.With(ctx).FetchAll(since, true)
|
||||
findATender := findatender.New(
|
||||
conf,
|
||||
conf.Sources.FTSMock.APIRoot,
|
||||
findatender.WithLogger(logger),
|
||||
)
|
||||
defer findATender.Shutdown()
|
||||
return findATender.With(ctx).FetchAll(since, true)
|
||||
})
|
||||
if errors.Is(err, findatender.ErrNoReleases) {
|
||||
return ErrNoNewReleases
|
||||
|
||||
@@ -15,10 +15,13 @@ const sleepDuration = 60 * time.Second
|
||||
// runSleeper runs the sleeper test source until ctx is cancelled: every
|
||||
// configured interval it simulates work by sleeping for sleepDuration.
|
||||
func runSleeper(
|
||||
ctx context.Context, logger *slog.Logger, conf *config.JobSearchConfig, since time.Time,
|
||||
ctx context.Context,
|
||||
logger *slog.Logger,
|
||||
conf *config.JobSearchConfig,
|
||||
since time.Time,
|
||||
) error {
|
||||
blockShutdown := conf.Sources.Sleeper.BlockShutdown
|
||||
w := util.Worker{
|
||||
worker := util.Worker{
|
||||
Logger: logger,
|
||||
Name: "Sleeper",
|
||||
Interval: conf.Sources.Sleeper.Interval,
|
||||
@@ -30,7 +33,7 @@ func runSleeper(
|
||||
}
|
||||
},
|
||||
}
|
||||
return w.Run(ctx, since)
|
||||
return worker.Run(ctx, since)
|
||||
}
|
||||
|
||||
// sleep simulates work taking duration, returning early with ctx.Err() if ctx
|
||||
|
||||
+82
-81
@@ -46,7 +46,7 @@ type RequestOptions struct {
|
||||
AllowRateLimit bool
|
||||
}
|
||||
|
||||
// ---------- Utility methods for handling API irregularities -----------
|
||||
// ---------- Utility methods for handling API irregularities ----------
|
||||
|
||||
// safeReleasePackage is ocds.ReleasePackage with the releases left undecoded,
|
||||
// so safeUnpackReleases can repair or skip each one.
|
||||
@@ -65,14 +65,14 @@ type safeReleasePackage struct {
|
||||
// dateOnly matches a calendar date with no time of day: "2006-10-02".
|
||||
var dateOnly = regexp.MustCompile(`^\d{4}-\d{2}-\d{2}$`)
|
||||
|
||||
// unmarshalLenientDates decodes raw json into v like json.Unmarshal, after
|
||||
// unmarshalLenientDates decodes raw json into target like json.Unmarshal, after
|
||||
// repairing date fields that time.Time cannot parse.
|
||||
func unmarshalLenientDates(raw json.RawMessage, v any) error {
|
||||
func unmarshalLenientDates(raw json.RawMessage, target any) error {
|
||||
fixed, err := repairDates(raw)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return json.Unmarshal(fixed, v)
|
||||
return json.Unmarshal(fixed, target)
|
||||
}
|
||||
|
||||
// repairDates returns a copy of raw in which every date field holding a
|
||||
@@ -85,15 +85,15 @@ func unmarshalLenientDates(raw json.RawMessage, v any) error {
|
||||
// A date field is one whose name is "date" or starts with "date", contains
|
||||
// "Date", or ends with "Deadline".
|
||||
func repairDates(raw json.RawMessage) (json.RawMessage, error) {
|
||||
dec := json.NewDecoder(bytes.NewReader(raw))
|
||||
dec.UseNumber() // keep numbers exactly as written, not as float64
|
||||
decoder := json.NewDecoder(bytes.NewReader(raw))
|
||||
decoder.UseNumber() // keep numbers exactly as written, not as float64
|
||||
|
||||
var doc any
|
||||
if err := dec.Decode(&doc); err != nil {
|
||||
if err := decoder.Decode(&doc); err != nil {
|
||||
return nil, fmt.Errorf("repairing dates: %w", err)
|
||||
}
|
||||
|
||||
if _, err := dec.Token(); !errors.Is(err, io.EOF) {
|
||||
if _, err := decoder.Token(); !errors.Is(err, io.EOF) {
|
||||
return nil, errors.New(
|
||||
"repairing dates: unexpected data after the JSON value",
|
||||
)
|
||||
@@ -106,31 +106,31 @@ func repairDates(raw json.RawMessage) (json.RawMessage, error) {
|
||||
return json.Marshal(doc)
|
||||
}
|
||||
|
||||
// recursiveRepairDates fixes date fields in v in place and reports whether it
|
||||
// recursiveRepairDates fixes date fields in value in place and reports whether it
|
||||
// changed anything.
|
||||
func recursiveRepairDates(v any) bool {
|
||||
func recursiveRepairDates(value any) bool {
|
||||
changed := false
|
||||
switch v := v.(type) {
|
||||
switch typed := value.(type) {
|
||||
case map[string]any:
|
||||
for key, val := range v {
|
||||
if s, ok := val.(string); ok && isDateField(key) {
|
||||
for key, fieldValue := range typed {
|
||||
if text, ok := fieldValue.(string); ok && isDateField(key) {
|
||||
switch {
|
||||
case dateOnly.MatchString(s):
|
||||
v[key] = s + "T00:00:00Z"
|
||||
case dateOnly.MatchString(text):
|
||||
typed[key] = text + "T00:00:00Z"
|
||||
changed = true
|
||||
case s == "":
|
||||
v[key] = nil
|
||||
case text == "":
|
||||
typed[key] = nil
|
||||
changed = true
|
||||
}
|
||||
continue
|
||||
}
|
||||
if recursiveRepairDates(val) {
|
||||
if recursiveRepairDates(fieldValue) {
|
||||
changed = true
|
||||
}
|
||||
}
|
||||
case []any:
|
||||
for _, el := range v {
|
||||
if recursiveRepairDates(el) {
|
||||
for _, element := range typed {
|
||||
if recursiveRepairDates(element) {
|
||||
changed = true
|
||||
}
|
||||
}
|
||||
@@ -152,35 +152,35 @@ func isDateField(name string) bool {
|
||||
// value that wouldn't fit the release struct) doesn't matter. Fields before
|
||||
// them only need to be valid JSON.
|
||||
func releaseIDs(raw json.RawMessage) (ocid, id string, err error) {
|
||||
dec := json.NewDecoder(bytes.NewReader(raw))
|
||||
dec.UseNumber() // a numeric id keeps its exact digits
|
||||
decoder := json.NewDecoder(bytes.NewReader(raw))
|
||||
decoder.UseNumber() // a numeric id keeps its exact digits
|
||||
|
||||
if tok, err := dec.Token(); err != nil {
|
||||
if token, err := decoder.Token(); err != nil {
|
||||
return "", "", fmt.Errorf("reading release: %w", err)
|
||||
} else if tok != json.Delim('{') {
|
||||
} else if token != json.Delim('{') {
|
||||
return "", "", errors.New("reading release: not a JSON object")
|
||||
}
|
||||
|
||||
var haveOCID, haveID bool
|
||||
for dec.More() {
|
||||
tok, err := dec.Token()
|
||||
for decoder.More() {
|
||||
token, err := decoder.Token()
|
||||
if err != nil {
|
||||
return ocid, id, fmt.Errorf("reading release: %w", err)
|
||||
}
|
||||
switch key, _ := tok.(string); key {
|
||||
switch key, _ := token.(string); key {
|
||||
case "ocid":
|
||||
if err := dec.Decode(&ocid); err != nil {
|
||||
if err := decoder.Decode(&ocid); err != nil {
|
||||
return ocid, id, fmt.Errorf("reading release ocid: %w", err)
|
||||
}
|
||||
haveOCID = true
|
||||
case "id":
|
||||
if id, err = decodeReleaseID(dec); err != nil {
|
||||
if id, err = decodeReleaseID(decoder); err != nil {
|
||||
return ocid, id, err
|
||||
}
|
||||
haveID = true
|
||||
default:
|
||||
var skip json.RawMessage // must be valid JSON, but is otherwise ignored
|
||||
if err := dec.Decode(&skip); err != nil {
|
||||
if err := decoder.Decode(&skip); err != nil {
|
||||
return ocid, id, fmt.Errorf("reading release field %q: %w", key, err)
|
||||
}
|
||||
}
|
||||
@@ -194,18 +194,18 @@ func releaseIDs(raw json.RawMessage) (ocid, id string, err error) {
|
||||
|
||||
// decodeReleaseID decodes a release id. OCDS ids are strings, but a number is
|
||||
// tolerated and kept as written.
|
||||
func decodeReleaseID(dec *json.Decoder) (string, error) {
|
||||
var v any
|
||||
if err := dec.Decode(&v); err != nil {
|
||||
func decodeReleaseID(decoder *json.Decoder) (string, error) {
|
||||
var decoded any
|
||||
if err := decoder.Decode(&decoded); err != nil {
|
||||
return "", fmt.Errorf("reading release id: %w", err)
|
||||
}
|
||||
switch v := v.(type) {
|
||||
switch typed := decoded.(type) {
|
||||
case string:
|
||||
return v, nil
|
||||
return typed, nil
|
||||
case json.Number:
|
||||
return v.String(), nil
|
||||
return typed.String(), nil
|
||||
default:
|
||||
return "", fmt.Errorf("reading release id: unexpected %T", v)
|
||||
return "", fmt.Errorf("reading release id: unexpected %T", typed)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -227,48 +227,49 @@ func missingIDsError(haveOCID, haveID bool) error {
|
||||
//
|
||||
// This method takes ownership of the response body and closes it when done.
|
||||
func safeUnpackReleases(
|
||||
logger *slog.Logger, response *resty.Response,
|
||||
logger *slog.Logger,
|
||||
response *resty.Response,
|
||||
) (*ocds.ReleasePackage, error) {
|
||||
var pkg safeReleasePackage
|
||||
var releasePackage safeReleasePackage
|
||||
|
||||
// Take ownership of the response body.
|
||||
defer response.Body.Close() //nolint:errcheck // read only
|
||||
|
||||
decoder := json.NewDecoder(response.Body)
|
||||
if err := decoder.Decode(&pkg); err != nil {
|
||||
if err := decoder.Decode(&releasePackage); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
out := &ocds.ReleasePackage{
|
||||
URI: pkg.URI,
|
||||
Version: pkg.Version,
|
||||
Extensions: pkg.Extensions,
|
||||
Publisher: pkg.Publisher,
|
||||
License: pkg.License,
|
||||
PublicationPolicy: pkg.PublicationPolicy,
|
||||
Links: pkg.Links,
|
||||
URI: releasePackage.URI,
|
||||
Version: releasePackage.Version,
|
||||
Extensions: releasePackage.Extensions,
|
||||
Publisher: releasePackage.Publisher,
|
||||
License: releasePackage.License,
|
||||
PublicationPolicy: releasePackage.PublicationPolicy,
|
||||
Links: releasePackage.Links,
|
||||
}
|
||||
|
||||
// Correct PublishedDate of "" and "2006-01-02"
|
||||
if pkg.PublishedDate != nil && len(*pkg.PublishedDate) > 0 {
|
||||
t, err := time.Parse(time.RFC3339, *pkg.PublishedDate)
|
||||
if releasePackage.PublishedDate != nil && len(*releasePackage.PublishedDate) > 0 {
|
||||
published, err := time.Parse(time.RFC3339, *releasePackage.PublishedDate)
|
||||
if err != nil {
|
||||
t, err = time.Parse(time.DateOnly, *pkg.PublishedDate)
|
||||
published, err = time.Parse(time.DateOnly, *releasePackage.PublishedDate)
|
||||
}
|
||||
if err == nil {
|
||||
out.PublishedDate = &t
|
||||
out.PublishedDate = &published
|
||||
}
|
||||
}
|
||||
|
||||
for _, rawRel := range pkg.Releases {
|
||||
for _, rawRelease := range releasePackage.Releases {
|
||||
var release ocds.Release
|
||||
err := unmarshalLenientDates(rawRel, &release)
|
||||
err := unmarshalLenientDates(rawRelease, &release)
|
||||
if err != nil {
|
||||
ocid, release_id, _ := releaseIDs(rawRel)
|
||||
ocid, releaseID, _ := releaseIDs(rawRelease)
|
||||
logger.Warn(
|
||||
"skipping unreadable release",
|
||||
"ocid", ocid,
|
||||
"release_id", release_id,
|
||||
"release_id", releaseID,
|
||||
"error", err,
|
||||
)
|
||||
continue
|
||||
@@ -279,7 +280,7 @@ func safeUnpackReleases(
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// ---------- API interaction methods -----------
|
||||
// ---------- API interaction methods ----------
|
||||
|
||||
// rateSleep sleeps the fetch worker until either the context is cancelled
|
||||
// (shutdown), returning ctx.Err(), or the duration has expired. The duration
|
||||
@@ -303,21 +304,21 @@ func closeBody(res *resty.Response) {
|
||||
}
|
||||
}
|
||||
|
||||
// fetchURI performs a single GET of uri, using ft's context, and traces it at
|
||||
// fetchURI performs a single GET of uri, using findATender's context, and traces it at
|
||||
// debug level. Unlike the method of the same name it does not check the
|
||||
// response status.
|
||||
func fetchURI(ft *FindATender, uri string) (*resty.Response, error) {
|
||||
withTrace := ft.config.Logging.LogLevel() <= slog.LevelDebug
|
||||
logger := ft.logger()
|
||||
func fetchURI(findATender *FindATender, uri string) (*resty.Response, error) {
|
||||
withTrace := findATender.config.Logging.LogLevel() <= slog.LevelDebug
|
||||
logger := findATender.logger()
|
||||
|
||||
logger.Info("fetching", "uri", uri)
|
||||
|
||||
ctx := ft.ctx
|
||||
ctx := findATender.ctx
|
||||
if ctx == nil {
|
||||
ctx = context.Background()
|
||||
}
|
||||
|
||||
req := ft.httpClient.R()
|
||||
req := findATender.httpClient.R()
|
||||
req.SetContext(ctx)
|
||||
res, err := req.SetTrace(withTrace).Get(uri)
|
||||
|
||||
@@ -373,13 +374,13 @@ func (ft *FindATender) fetchURI(
|
||||
return nil, ErrRateLimitExceeded
|
||||
}
|
||||
|
||||
retrySecStr := res.Header().Get("Retry-After")
|
||||
retrySec, err := strconv.ParseInt(retrySecStr, 10, 32)
|
||||
retryAfterText := res.Header().Get("Retry-After")
|
||||
retryAfterSeconds, err := strconv.ParseInt(retryAfterText, 10, 32)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("unknown retry-after value: %w", err)
|
||||
}
|
||||
|
||||
retryAfter := time.Duration(retrySec) * time.Second
|
||||
retryAfter := time.Duration(retryAfterSeconds) * time.Second
|
||||
ft.logger().Warn("api rate limit hit, waiting as asked", "retry_after", retryAfter)
|
||||
if rateSleep(ctx, retryAfter) != nil {
|
||||
return nil, ErrFetchCancelled
|
||||
@@ -429,13 +430,13 @@ func (ft *FindATender) fetchWithOpts(
|
||||
// ErrNoNext if there is no "Next" link; the caller can decide whether asking
|
||||
// for it was an error.
|
||||
func (ft *FindATender) fetchNext(
|
||||
pkg *ocds.ReleasePackage,
|
||||
releasePackage *ocds.ReleasePackage,
|
||||
allowRateLimit bool,
|
||||
) (*resty.Response, error) {
|
||||
if pkg.Links == nil || len(pkg.Links.Next) == 0 {
|
||||
if releasePackage.Links == nil || len(releasePackage.Links.Next) == 0 {
|
||||
return nil, ErrNoNext
|
||||
}
|
||||
return ft.fetchURI(pkg.Links.Next, allowRateLimit)
|
||||
return ft.fetchURI(releasePackage.Links.Next, allowRateLimit)
|
||||
}
|
||||
|
||||
// fetchOnce makes one call on the API. It does not follow any "Next" links.
|
||||
@@ -445,16 +446,16 @@ func (ft *FindATender) fetchOnce(opts RequestOptions) ([]ocds.Release, error) {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
pkg, err := safeUnpackReleases(ft.logger(), res)
|
||||
releasePackage, err := safeUnpackReleases(ft.logger(), res)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if len(pkg.Releases) == 0 {
|
||||
if len(releasePackage.Releases) == 0 {
|
||||
return nil, ErrNoReleases
|
||||
}
|
||||
|
||||
return pkg.Releases, nil
|
||||
return releasePackage.Releases, nil
|
||||
}
|
||||
|
||||
// fetchAll makes a call on the API. If there are any "Next" links, it calls
|
||||
@@ -467,25 +468,25 @@ func (ft *FindATender) fetchAll(opts RequestOptions) ([]ocds.Release, error) {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
pkg, err := safeUnpackReleases(ft.logger(), res)
|
||||
releasePackage, err := safeUnpackReleases(ft.logger(), res)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
releases = append(releases, pkg.Releases...)
|
||||
releases = append(releases, releasePackage.Releases...)
|
||||
|
||||
// Next links already followed, so a page that links back to an earlier
|
||||
// one cannot loop forever.
|
||||
seen := make(map[string]bool)
|
||||
for {
|
||||
if pkg.Links != nil && len(pkg.Links.Next) > 0 {
|
||||
if seen[pkg.Links.Next] {
|
||||
ft.logger().Warn("stopping at repeated next link", "link", pkg.Links.Next)
|
||||
if releasePackage.Links != nil && len(releasePackage.Links.Next) > 0 {
|
||||
if seen[releasePackage.Links.Next] {
|
||||
ft.logger().Warn("stopping at repeated next link", "link", releasePackage.Links.Next)
|
||||
break
|
||||
}
|
||||
seen[pkg.Links.Next] = true
|
||||
seen[releasePackage.Links.Next] = true
|
||||
}
|
||||
|
||||
res, err := ft.fetchNext(pkg, opts.AllowRateLimit)
|
||||
res, err := ft.fetchNext(releasePackage, opts.AllowRateLimit)
|
||||
if err == ErrNoNext {
|
||||
break
|
||||
}
|
||||
@@ -493,11 +494,11 @@ func (ft *FindATender) fetchAll(opts RequestOptions) ([]ocds.Release, error) {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
pkg, err = safeUnpackReleases(ft.logger(), res)
|
||||
releasePackage, err = safeUnpackReleases(ft.logger(), res)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
releases = append(releases, pkg.Releases...)
|
||||
releases = append(releases, releasePackage.Releases...)
|
||||
}
|
||||
|
||||
if len(releases) == 0 {
|
||||
@@ -505,4 +506,4 @@ func (ft *FindATender) fetchAll(opts RequestOptions) ([]ocds.Release, error) {
|
||||
}
|
||||
|
||||
return releases, nil
|
||||
}
|
||||
}
|
||||
@@ -22,7 +22,11 @@ import (
|
||||
)
|
||||
|
||||
func testConfig(level string) *config.JobSearchConfig {
|
||||
return &config.JobSearchConfig{Logging: config.JSCLogging{Level: level}}
|
||||
return &config.JobSearchConfig{
|
||||
Logging: config.JSCLogging{
|
||||
Level: level,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// quietLogs discards slog output for the duration of the test, or sends it
|
||||
@@ -34,7 +38,9 @@ func quietLogs(t *testing.T, h slog.Handler) {
|
||||
}
|
||||
prev := slog.Default()
|
||||
slog.SetDefault(slog.New(h))
|
||||
t.Cleanup(func() { slog.SetDefault(prev) })
|
||||
t.Cleanup(func() {
|
||||
slog.SetDefault(prev)
|
||||
})
|
||||
}
|
||||
|
||||
// newTestFT starts h on a test server and returns a FindATender pointed at it.
|
||||
@@ -92,7 +98,10 @@ func TestFetchWithOptsBuildsURI(t *testing.T) {
|
||||
})
|
||||
|
||||
since := time.Date(2026, 10, 8, 9, 30, 15, 0, time.UTC)
|
||||
res, err := ft.fetchWithOpts(RequestOptions{Since: since, Limit: 7})
|
||||
res, err := ft.fetchWithOpts(RequestOptions{
|
||||
Since: since,
|
||||
Limit: 7,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -104,7 +113,9 @@ func TestFetchWithOptsBuildsURI(t *testing.T) {
|
||||
t.Errorf("query = %q", gotQuery)
|
||||
}
|
||||
|
||||
res, err = ft.fetchWithOpts(RequestOptions{Since: since})
|
||||
res, err = ft.fetchWithOpts(RequestOptions{
|
||||
Since: since,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -268,7 +279,8 @@ func TestFetchURIRateLimitThenError(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestFetchURIConnectionError(t *testing.T) {
|
||||
ft, srv := newTestFT(t, "warn", func(w http.ResponseWriter, r *http.Request) {})
|
||||
ft, srv := newTestFT(t, "warn", func(w http.ResponseWriter, r *http.Request) {
|
||||
})
|
||||
srv.Close()
|
||||
ft.httpClient.SetRetryCount(0)
|
||||
if _, err := ft.fetchURI(srv.URL, true); err == nil {
|
||||
@@ -290,10 +302,14 @@ func TestFetchURICancelledContext(t *testing.T) {
|
||||
func TestFetchURIDebugTrace(t *testing.T) {
|
||||
handlers := map[string]func(io.Writer) slog.Handler{
|
||||
"text": func(w io.Writer) slog.Handler {
|
||||
return slog.NewTextHandler(w, &slog.HandlerOptions{Level: slog.LevelDebug})
|
||||
return slog.NewTextHandler(w, &slog.HandlerOptions{
|
||||
Level: slog.LevelDebug,
|
||||
})
|
||||
},
|
||||
"json": func(w io.Writer) slog.Handler {
|
||||
return slog.NewJSONHandler(w, &slog.HandlerOptions{Level: slog.LevelDebug})
|
||||
return slog.NewJSONHandler(w, &slog.HandlerOptions{
|
||||
Level: slog.LevelDebug,
|
||||
})
|
||||
},
|
||||
}
|
||||
for name, mk := range handlers {
|
||||
@@ -305,7 +321,9 @@ func TestFetchURIDebugTrace(t *testing.T) {
|
||||
quietLogs(t, mk(&logs))
|
||||
|
||||
// The body must still be readable after the debug logging.
|
||||
releases, err := ft.fetchOnce(RequestOptions{Since: time.Now()})
|
||||
releases, err := ft.fetchOnce(RequestOptions{
|
||||
Since: time.Now(),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -325,7 +343,8 @@ func TestFetchURIDebugTrace(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestFetchURIDebugTraceConnectionError(t *testing.T) {
|
||||
ft, srv := newTestFT(t, "debug", func(w http.ResponseWriter, r *http.Request) {})
|
||||
ft, srv := newTestFT(t, "debug", func(w http.ResponseWriter, r *http.Request) {
|
||||
})
|
||||
srv.Close()
|
||||
ft.httpClient.SetRetryCount(0)
|
||||
if _, err := fetchURI(ft, srv.URL); err == nil {
|
||||
@@ -341,8 +360,10 @@ func TestFetchNext(t *testing.T) {
|
||||
})
|
||||
|
||||
for name, pkg := range map[string]*ocds.ReleasePackage{
|
||||
"nil links": {},
|
||||
"empty next": {Links: &ocds.ReleasePackageLinks{Prev: srv.URL}},
|
||||
"nil links": {},
|
||||
"empty next": {Links: &ocds.ReleasePackageLinks{
|
||||
Prev: srv.URL,
|
||||
}},
|
||||
} {
|
||||
if _, err := ft.fetchNext(pkg, true); !errors.Is(err, ErrNoNext) {
|
||||
t.Errorf("%s: err = %v, want ErrNoNext", name, err)
|
||||
@@ -350,7 +371,11 @@ func TestFetchNext(t *testing.T) {
|
||||
}
|
||||
|
||||
res, err := ft.fetchNext(
|
||||
&ocds.ReleasePackage{Links: &ocds.ReleasePackageLinks{Next: srv.URL + "/page2"}},
|
||||
&ocds.ReleasePackage{
|
||||
Links: &ocds.ReleasePackageLinks{
|
||||
Next: srv.URL + "/page2",
|
||||
},
|
||||
},
|
||||
true,
|
||||
)
|
||||
if err != nil {
|
||||
@@ -372,7 +397,9 @@ func responseFor(t *testing.T, body string) *resty.Response {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { res.Body.Close() })
|
||||
t.Cleanup(func() {
|
||||
res.Body.Close()
|
||||
})
|
||||
return res
|
||||
}
|
||||
|
||||
@@ -481,11 +508,14 @@ func pagedServer(t *testing.T, pages ...[]string) (*FindATender, *atomic.Int32)
|
||||
}
|
||||
|
||||
func TestFetchOnce(t *testing.T) {
|
||||
ft, hits := pagedServer(t,
|
||||
ft, hits := pagedServer(
|
||||
t,
|
||||
[]string{releaseJSON("ocds-1", "1"), releaseJSON("ocds-2", "1")},
|
||||
[]string{releaseJSON("ocds-3", "1")},
|
||||
)
|
||||
releases, err := ft.fetchOnce(RequestOptions{Since: time.Now()})
|
||||
releases, err := ft.fetchOnce(RequestOptions{
|
||||
Since: time.Now(),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -499,14 +529,18 @@ func TestFetchOnce(t *testing.T) {
|
||||
|
||||
func TestFetchOnceErrors(t *testing.T) {
|
||||
ft, _ := pagedServer(t, []string{})
|
||||
if _, err := ft.fetchOnce(RequestOptions{Since: time.Now()}); !errors.Is(err, ErrNoReleases) {
|
||||
if _, err := ft.fetchOnce(RequestOptions{
|
||||
Since: time.Now(),
|
||||
}); !errors.Is(err, ErrNoReleases) {
|
||||
t.Errorf("empty page: err = %v, want ErrNoReleases", err)
|
||||
}
|
||||
|
||||
ft, _ = newTestFT(t, "warn", func(w http.ResponseWriter, r *http.Request) {
|
||||
io.WriteString(w, "not json")
|
||||
})
|
||||
if _, err := ft.fetchOnce(RequestOptions{Since: time.Now()}); err == nil {
|
||||
if _, err := ft.fetchOnce(RequestOptions{
|
||||
Since: time.Now(),
|
||||
}); err == nil {
|
||||
t.Error("bad body: expected an error")
|
||||
}
|
||||
|
||||
@@ -514,7 +548,9 @@ func TestFetchOnceErrors(t *testing.T) {
|
||||
w.WriteHeader(http.StatusTooManyRequests)
|
||||
})
|
||||
if _, err := ft.fetchOnce(
|
||||
RequestOptions{Since: time.Now()},
|
||||
RequestOptions{
|
||||
Since: time.Now(),
|
||||
},
|
||||
); !errors.Is(
|
||||
err,
|
||||
ErrRateLimitExceeded,
|
||||
@@ -524,12 +560,15 @@ func TestFetchOnceErrors(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestFetchAll(t *testing.T) {
|
||||
ft, hits := pagedServer(t,
|
||||
ft, hits := pagedServer(
|
||||
t,
|
||||
[]string{releaseJSON("ocds-1", "1"), releaseJSON("ocds-2", "1")},
|
||||
[]string{},
|
||||
[]string{releaseJSON("ocds-1", "2")},
|
||||
)
|
||||
releases, err := ft.fetchAll(RequestOptions{Since: time.Now()})
|
||||
releases, err := ft.fetchAll(RequestOptions{
|
||||
Since: time.Now(),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -547,7 +586,9 @@ func TestFetchAll(t *testing.T) {
|
||||
|
||||
func TestFetchAllErrors(t *testing.T) {
|
||||
ft, _ := pagedServer(t, []string{}, []string{})
|
||||
if _, err := ft.fetchAll(RequestOptions{Since: time.Now()}); !errors.Is(err, ErrNoReleases) {
|
||||
if _, err := ft.fetchAll(RequestOptions{
|
||||
Since: time.Now(),
|
||||
}); !errors.Is(err, ErrNoReleases) {
|
||||
t.Errorf("all pages empty: err = %v, want ErrNoReleases", err)
|
||||
}
|
||||
|
||||
@@ -556,7 +597,9 @@ func TestFetchAllErrors(t *testing.T) {
|
||||
w.WriteHeader(http.StatusTooManyRequests)
|
||||
})
|
||||
if _, err := ft.fetchAll(
|
||||
RequestOptions{Since: time.Now()},
|
||||
RequestOptions{
|
||||
Since: time.Now(),
|
||||
},
|
||||
); !errors.Is(
|
||||
err,
|
||||
ErrRateLimitExceeded,
|
||||
@@ -567,14 +610,20 @@ func TestFetchAllErrors(t *testing.T) {
|
||||
ft, _ = newTestFT(t, "warn", func(w http.ResponseWriter, r *http.Request) {
|
||||
io.WriteString(w, "not json")
|
||||
})
|
||||
if _, err := ft.fetchAll(RequestOptions{Since: time.Now()}); err == nil {
|
||||
if _, err := ft.fetchAll(RequestOptions{
|
||||
Since: time.Now(),
|
||||
}); err == nil {
|
||||
t.Error("first page bad body: expected an error")
|
||||
}
|
||||
|
||||
// A later page fails, either to fetch or to decode.
|
||||
for name, fail := range map[string]http.HandlerFunc{
|
||||
"rate limited": func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusTooManyRequests) },
|
||||
"bad body": func(w http.ResponseWriter, r *http.Request) { io.WriteString(w, "not json") },
|
||||
"rate limited": func(w http.ResponseWriter, r *http.Request) {
|
||||
w.WriteHeader(http.StatusTooManyRequests)
|
||||
},
|
||||
"bad body": func(w http.ResponseWriter, r *http.Request) {
|
||||
io.WriteString(w, "not json")
|
||||
},
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
var srvURL string
|
||||
@@ -586,7 +635,9 @@ func TestFetchAllErrors(t *testing.T) {
|
||||
fail(w, r)
|
||||
})
|
||||
srvURL = srv.URL
|
||||
if _, err := ft.fetchAll(RequestOptions{Since: time.Now()}); err == nil {
|
||||
if _, err := ft.fetchAll(RequestOptions{
|
||||
Since: time.Now(),
|
||||
}); err == nil {
|
||||
t.Error("expected an error from the second page")
|
||||
}
|
||||
})
|
||||
@@ -597,7 +648,9 @@ func TestFetchAllErrors(t *testing.T) {
|
||||
func TestFetchAllRepeatedNextLink(t *testing.T) {
|
||||
tests := map[string]func(base string, page string) string{
|
||||
// page 0 -> page 1 -> page 1 ...
|
||||
"self link": func(base, page string) string { return base + "/next?page=1" },
|
||||
"self link": func(base, page string) string {
|
||||
return base + "/next?page=1"
|
||||
},
|
||||
// page 0 -> page 1 -> page 2 -> page 1 ...
|
||||
"cycle": func(base, page string) string {
|
||||
if page == "1" {
|
||||
@@ -623,7 +676,9 @@ func TestFetchAllRepeatedNextLink(t *testing.T) {
|
||||
})
|
||||
srvURL = srv.URL
|
||||
|
||||
releases, err := ft.fetchAll(RequestOptions{Since: time.Now()})
|
||||
releases, err := ft.fetchAll(RequestOptions{
|
||||
Since: time.Now(),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -668,7 +723,9 @@ func TestUnmarshalLenientDates(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
midnight := func(y int, m time.Month, d int) time.Time { return time.Date(y, m, d, 0, 0, 0, 0, time.UTC) }
|
||||
midnight := func(y int, m time.Month, d int) time.Time {
|
||||
return time.Date(y, m, d, 0, 0, 0, 0, time.UTC)
|
||||
}
|
||||
if !got.Date.Equal(midnight(2006, 10, 2)) {
|
||||
t.Errorf("date = %v", got.Date)
|
||||
}
|
||||
|
||||
@@ -35,17 +35,17 @@ func WithLogger(logger *slog.Logger) Option {
|
||||
// 60 seconds and are retried once. Call Shutdown when finished with it. It
|
||||
// logs to slog.Default() unless given WithLogger.
|
||||
func New(conf *config.JobSearchConfig, apiRoot string, opts ...Option) *FindATender {
|
||||
ft := &FindATender{
|
||||
findATender := &FindATender{
|
||||
config: conf,
|
||||
apiRoot: apiRoot,
|
||||
}
|
||||
for _, opt := range opts {
|
||||
opt(ft)
|
||||
opt(findATender)
|
||||
}
|
||||
ft.httpClient = resty.New().
|
||||
findATender.httpClient = resty.New().
|
||||
SetTimeout(60 * time.Duration(time.Second)).
|
||||
SetRetryCount(1)
|
||||
return ft
|
||||
return findATender
|
||||
}
|
||||
|
||||
// logger returns the logger the client logs to, tagged with its component.
|
||||
|
||||
@@ -79,7 +79,8 @@ func opportunityIDs(ops []*types.Opportunity) []string {
|
||||
}
|
||||
|
||||
func TestFetchOncePublic(t *testing.T) {
|
||||
ft, _ := pagedServer(t,
|
||||
ft, _ := pagedServer(
|
||||
t,
|
||||
[]string{releaseJSON("ocds-1", "1"), releaseJSON("ocds-1", "2"), releaseJSON("ocds-2", "1")},
|
||||
[]string{releaseJSON("ocds-3", "1")},
|
||||
)
|
||||
@@ -98,7 +99,8 @@ func TestFetchOncePublic(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestFetchAllPublic(t *testing.T) {
|
||||
ft, _ := pagedServer(t,
|
||||
ft, _ := pagedServer(
|
||||
t,
|
||||
[]string{releaseJSON("ocds-1", "1"), releaseJSON("ocds-2", "1")},
|
||||
[]string{releaseJSON("ocds-1", "2"), releaseJSON("ocds-3", "1")},
|
||||
)
|
||||
|
||||
@@ -48,8 +48,8 @@ func ocdsGetClassifications(release *ocds.Release) []types.ClassificationCode {
|
||||
add := func(classif *ocds.Classification) {
|
||||
newClassif := ocdsConvertClassification(classif)
|
||||
found := slices.ContainsFunc(
|
||||
classifications, func(r types.ClassificationCode) bool {
|
||||
return r.Code == newClassif.Code
|
||||
classifications, func(code types.ClassificationCode) bool {
|
||||
return code.Code == newClassif.Code
|
||||
},
|
||||
)
|
||||
if !found {
|
||||
@@ -59,11 +59,11 @@ func ocdsGetClassifications(release *ocds.Release) []types.ClassificationCode {
|
||||
if release.Tender.Classification != nil {
|
||||
add(release.Tender.Classification)
|
||||
}
|
||||
for _, val := range release.Tender.Items {
|
||||
if val.Classification != nil {
|
||||
add(val.Classification)
|
||||
for _, item := range release.Tender.Items {
|
||||
if item.Classification != nil {
|
||||
add(item.Classification)
|
||||
}
|
||||
for _, classif := range val.AdditionalClassifications {
|
||||
for _, classif := range item.AdditionalClassifications {
|
||||
add(&classif)
|
||||
}
|
||||
}
|
||||
@@ -113,15 +113,15 @@ func ocdsIsVCSESuitable(release *ocds.Release) bool {
|
||||
// re-encoded copy of the whole release as JSON.
|
||||
func ocdsRelease(release *ocds.Release) types.OpportunityRelease {
|
||||
tags := make([]types.OpportunityReleaseTag, 0)
|
||||
for _, val := range release.Tag {
|
||||
for _, tag := range release.Tag {
|
||||
tags = append(tags, types.OpportunityReleaseTag{
|
||||
UUID: release.OCID,
|
||||
ReleaseID: release.ID,
|
||||
Tag: val,
|
||||
Tag: tag,
|
||||
})
|
||||
}
|
||||
|
||||
rel := types.OpportunityRelease{
|
||||
converted := types.OpportunityRelease{
|
||||
UUID: release.OCID,
|
||||
ReleaseID: release.ID,
|
||||
Tags: tags,
|
||||
@@ -130,16 +130,16 @@ func ocdsRelease(release *ocds.Release) types.OpportunityRelease {
|
||||
|
||||
reencoded, err := json.Marshal(release)
|
||||
if err == nil {
|
||||
rel.JSON = string(reencoded)
|
||||
converted.JSON = string(reencoded)
|
||||
}
|
||||
|
||||
return rel
|
||||
return converted
|
||||
}
|
||||
|
||||
// convertOne builds an opportunity from a single release. Convert passes it
|
||||
// each ocid's latest release, as that has the current state of the tender.
|
||||
func convertOne(release *ocds.Release) *types.Opportunity {
|
||||
t := &types.Opportunity{
|
||||
opportunity := &types.Opportunity{
|
||||
UUID: release.OCID,
|
||||
UUIDType: "ocid",
|
||||
BuyerId: ocdsGetBuyerId(release),
|
||||
@@ -148,29 +148,29 @@ func convertOne(release *ocds.Release) *types.Opportunity {
|
||||
}
|
||||
|
||||
if release.Tender != nil {
|
||||
t.Title = release.Tender.Title
|
||||
t.Status = release.Tender.Status
|
||||
t.Description = release.Tender.Description
|
||||
t.Classifications = ocdsGetClassifications(release)
|
||||
t.MainCategory = release.Tender.MainProcurementCategory
|
||||
t.ProcurementMethod = release.Tender.ProcurementMethod
|
||||
t.SMESuitable = ocdsIsSMESuitable(release)
|
||||
t.VCSESuitable = ocdsIsVCSESuitable(release)
|
||||
opportunity.Title = release.Tender.Title
|
||||
opportunity.Status = release.Tender.Status
|
||||
opportunity.Description = release.Tender.Description
|
||||
opportunity.Classifications = ocdsGetClassifications(release)
|
||||
opportunity.MainCategory = release.Tender.MainProcurementCategory
|
||||
opportunity.ProcurementMethod = release.Tender.ProcurementMethod
|
||||
opportunity.SMESuitable = ocdsIsSMESuitable(release)
|
||||
opportunity.VCSESuitable = ocdsIsVCSESuitable(release)
|
||||
if release.Tender.TenderPeriod != nil {
|
||||
t.StartDate = release.Tender.TenderPeriod.StartDate
|
||||
t.EndDate = release.Tender.TenderPeriod.EndDate
|
||||
opportunity.StartDate = release.Tender.TenderPeriod.StartDate
|
||||
opportunity.EndDate = release.Tender.TenderPeriod.EndDate
|
||||
}
|
||||
if release.Tender.Value != nil && release.Tender.Value.Amount != nil {
|
||||
t.GrossValue = *release.Tender.Value.Amount
|
||||
opportunity.GrossValue = *release.Tender.Value.Amount
|
||||
}
|
||||
t.HasDocuments = len(release.Tender.Documents) > 0
|
||||
t.MultiLot = len(release.Tender.Lots) > 1
|
||||
opportunity.HasDocuments = len(release.Tender.Documents) > 0
|
||||
opportunity.MultiLot = len(release.Tender.Lots) > 1
|
||||
}
|
||||
|
||||
t.LatestReleaseDate = release.Date
|
||||
t.Releases = append(t.Releases, ocdsRelease(release))
|
||||
opportunity.LatestReleaseDate = release.Date
|
||||
opportunity.Releases = append(opportunity.Releases, ocdsRelease(release))
|
||||
|
||||
return t
|
||||
return opportunity
|
||||
}
|
||||
|
||||
// Convert groups releases by ocid into opportunities, in the order each ocid
|
||||
@@ -181,8 +181,8 @@ func Convert(releases []ocds.Release) []*types.Opportunity {
|
||||
var order []string
|
||||
byOCID := make(map[string][]*ocds.Release)
|
||||
|
||||
for idx := range releases {
|
||||
release := &releases[idx]
|
||||
for i := range releases {
|
||||
release := &releases[i]
|
||||
if _, found := byOCID[release.OCID]; !found {
|
||||
order = append(order, release.OCID)
|
||||
}
|
||||
@@ -193,8 +193,8 @@ func Convert(releases []ocds.Release) []*types.Opportunity {
|
||||
for _, ocid := range order {
|
||||
group := byOCID[ocid]
|
||||
|
||||
slices.SortStableFunc(group, func(a, b *ocds.Release) int {
|
||||
return a.Date.Compare(b.Date)
|
||||
slices.SortStableFunc(group, func(left, right *ocds.Release) int {
|
||||
return left.Date.Compare(right.Date)
|
||||
})
|
||||
|
||||
opportunity := convertOne(group[len(group)-1])
|
||||
|
||||
@@ -13,7 +13,12 @@ import (
|
||||
|
||||
func boolPtr(b bool) *bool { return &b }
|
||||
|
||||
func lotWith(s *ocds.Suitability) ocds.Lot { return ocds.Lot{ID: "1", Suitability: s} }
|
||||
func lotWith(s *ocds.Suitability) ocds.Lot {
|
||||
return ocds.Lot{
|
||||
ID: "1",
|
||||
Suitability: s,
|
||||
}
|
||||
}
|
||||
|
||||
func TestOcdsGetBuyer(t *testing.T) {
|
||||
tests := []struct {
|
||||
@@ -23,11 +28,16 @@ func TestOcdsGetBuyer(t *testing.T) {
|
||||
}{
|
||||
{"nil buyer", nil, "", ""},
|
||||
{"empty buyer", &ocds.OrganizationReference{}, "", ""},
|
||||
{"full buyer", &ocds.OrganizationReference{ID: "GB-1", Name: "Council"}, "GB-1", "Council"},
|
||||
{"full buyer", &ocds.OrganizationReference{
|
||||
ID: "GB-1",
|
||||
Name: "Council",
|
||||
}, "GB-1", "Council"},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
r := &ocds.Release{Buyer: tt.buyer}
|
||||
r := &ocds.Release{
|
||||
Buyer: tt.buyer,
|
||||
}
|
||||
if got := ocdsGetBuyerId(r); got != tt.wantID {
|
||||
t.Errorf("buyer id = %q, want %q", got, tt.wantID)
|
||||
}
|
||||
@@ -44,12 +54,34 @@ func TestOcdsConvertClassification(t *testing.T) {
|
||||
in ocds.Classification
|
||||
want types.ClassificationCode
|
||||
}{
|
||||
{"CPV", ocds.Classification{Scheme: "CPV", ID: "72000000", Description: "IT services"},
|
||||
types.ClassificationCode{IsCPV: true, Code: "72000000", Classification: "IT services"}},
|
||||
{"non-CPV", ocds.Classification{Scheme: "UNSPSC", ID: "43000000", Description: "IT"},
|
||||
types.ClassificationCode{IsCPV: false, Code: "43000000", Classification: "IT"}},
|
||||
{"scheme is case sensitive", ocds.Classification{Scheme: "cpv", ID: "1"},
|
||||
types.ClassificationCode{IsCPV: false, Code: "1"}},
|
||||
{"CPV", ocds.Classification{
|
||||
Scheme: "CPV",
|
||||
ID: "72000000",
|
||||
Description: "IT services",
|
||||
},
|
||||
types.ClassificationCode{
|
||||
IsCPV: true,
|
||||
Code: "72000000",
|
||||
Classification: "IT services",
|
||||
}},
|
||||
{"non-CPV", ocds.Classification{
|
||||
Scheme: "UNSPSC",
|
||||
ID: "43000000",
|
||||
Description: "IT",
|
||||
},
|
||||
types.ClassificationCode{
|
||||
IsCPV: false,
|
||||
Code: "43000000",
|
||||
Classification: "IT",
|
||||
}},
|
||||
{"scheme is case sensitive", ocds.Classification{
|
||||
Scheme: "cpv",
|
||||
ID: "1",
|
||||
},
|
||||
types.ClassificationCode{
|
||||
IsCPV: false,
|
||||
Code: "1",
|
||||
}},
|
||||
{"empty", ocds.Classification{}, types.ClassificationCode{}},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
@@ -63,7 +95,11 @@ func TestOcdsConvertClassification(t *testing.T) {
|
||||
|
||||
func TestOcdsGetClassifications(t *testing.T) {
|
||||
cpv := func(id string) ocds.Classification {
|
||||
return ocds.Classification{Scheme: "CPV", ID: ocds.ID(id), Description: "d" + id}
|
||||
return ocds.Classification{
|
||||
Scheme: "CPV",
|
||||
ID: ocds.ID(id),
|
||||
Description: "d" + id,
|
||||
}
|
||||
}
|
||||
codes := func(cs []types.ClassificationCode) []string {
|
||||
out := []string{}
|
||||
@@ -80,7 +116,9 @@ func TestOcdsGetClassifications(t *testing.T) {
|
||||
}{
|
||||
{"nil tender", nil, []string{}},
|
||||
{"no classifications", &ocds.Tender{}, []string{}},
|
||||
{"main only", &ocds.Tender{Classification: &main}, []string{"1"}},
|
||||
{"main only", &ocds.Tender{
|
||||
Classification: &main,
|
||||
}, []string{"1"}},
|
||||
{"main and item additionals, deduplicated", &ocds.Tender{
|
||||
Classification: &main,
|
||||
Items: []ocds.Item{
|
||||
@@ -95,7 +133,9 @@ func TestOcdsGetClassifications(t *testing.T) {
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := ocdsGetClassifications(&ocds.Release{Tender: tt.tender})
|
||||
got := ocdsGetClassifications(&ocds.Release{
|
||||
Tender: tt.tender,
|
||||
})
|
||||
if got == nil {
|
||||
t.Fatal("got nil slice, want non-nil")
|
||||
}
|
||||
@@ -107,7 +147,10 @@ func TestOcdsGetClassifications(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestOcdsGetClassificationsIncludesItemClassification(t *testing.T) {
|
||||
item := ocds.Classification{Scheme: "CPV", ID: "9"}
|
||||
item := ocds.Classification{
|
||||
Scheme: "CPV",
|
||||
ID: "9",
|
||||
}
|
||||
got := ocdsGetClassifications(&ocds.Release{Tender: &ocds.Tender{
|
||||
Items: []ocds.Item{{ID: "a", Classification: &item}},
|
||||
}})
|
||||
@@ -125,31 +168,49 @@ func TestOcdsSuitability(t *testing.T) {
|
||||
}{
|
||||
{"nil tender", nil, false, false},
|
||||
{"no lots", &ocds.Tender{}, false, false},
|
||||
{"lot without suitability", &ocds.Tender{Lots: []ocds.Lot{lotWith(nil)}}, false, false},
|
||||
{"lot without suitability", &ocds.Tender{
|
||||
Lots: []ocds.Lot{lotWith(nil)},
|
||||
}, false, false},
|
||||
{
|
||||
"suitability fields nil",
|
||||
&ocds.Tender{Lots: []ocds.Lot{lotWith(&ocds.Suitability{})}},
|
||||
&ocds.Tender{
|
||||
Lots: []ocds.Lot{lotWith(&ocds.Suitability{})},
|
||||
},
|
||||
false,
|
||||
false,
|
||||
},
|
||||
{"explicitly false", &ocds.Tender{Lots: []ocds.Lot{
|
||||
lotWith(&ocds.Suitability{SME: boolPtr(false), Vcse: boolPtr(false)}),
|
||||
lotWith(&ocds.Suitability{
|
||||
SME: boolPtr(false),
|
||||
Vcse: boolPtr(false),
|
||||
}),
|
||||
}}, false, false},
|
||||
{"SME only", &ocds.Tender{Lots: []ocds.Lot{
|
||||
lotWith(&ocds.Suitability{SME: boolPtr(true)}),
|
||||
lotWith(&ocds.Suitability{
|
||||
SME: boolPtr(true),
|
||||
}),
|
||||
}}, true, false},
|
||||
{"VCSE only", &ocds.Tender{Lots: []ocds.Lot{
|
||||
lotWith(&ocds.Suitability{Vcse: boolPtr(true)}),
|
||||
lotWith(&ocds.Suitability{
|
||||
Vcse: boolPtr(true),
|
||||
}),
|
||||
}}, false, true},
|
||||
{"any lot suffices", &ocds.Tender{Lots: []ocds.Lot{
|
||||
lotWith(nil),
|
||||
lotWith(&ocds.Suitability{SME: boolPtr(false)}),
|
||||
lotWith(&ocds.Suitability{SME: boolPtr(true), Vcse: boolPtr(true)}),
|
||||
lotWith(&ocds.Suitability{
|
||||
SME: boolPtr(false),
|
||||
}),
|
||||
lotWith(&ocds.Suitability{
|
||||
SME: boolPtr(true),
|
||||
Vcse: boolPtr(true),
|
||||
}),
|
||||
}}, true, true},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
r := &ocds.Release{Tender: tt.tender}
|
||||
r := &ocds.Release{
|
||||
Tender: tt.tender,
|
||||
}
|
||||
if got := ocdsIsSMESuitable(r); got != tt.wantSME {
|
||||
t.Errorf("SME = %v, want %v", got, tt.wantSME)
|
||||
}
|
||||
@@ -162,7 +223,12 @@ func TestOcdsSuitability(t *testing.T) {
|
||||
|
||||
func TestOcdsRelease(t *testing.T) {
|
||||
date := time.Date(2026, 10, 8, 12, 0, 0, 0, time.UTC)
|
||||
r := &ocds.Release{OCID: "ocds-x-1", ID: "001", Date: date, Tag: []string{"planning", "tender"}}
|
||||
r := &ocds.Release{
|
||||
OCID: "ocds-x-1",
|
||||
ID: "001",
|
||||
Date: date,
|
||||
Tag: []string{"planning", "tender"},
|
||||
}
|
||||
|
||||
got := ocdsRelease(r)
|
||||
if got.UUID != "ocds-x-1" || got.ReleaseID != "001" || !got.ReleaseDate.Equal(date) {
|
||||
@@ -193,12 +259,19 @@ func TestConvertOne(t *testing.T) {
|
||||
start := time.Date(2026, 10, 1, 0, 0, 0, 0, time.UTC)
|
||||
end := time.Date(2026, 11, 1, 0, 0, 0, 0, time.UTC)
|
||||
amount := 125000.5
|
||||
main := ocds.Classification{Scheme: "CPV", ID: "72000000", Description: "IT"}
|
||||
main := ocds.Classification{
|
||||
Scheme: "CPV",
|
||||
ID: "72000000",
|
||||
Description: "IT",
|
||||
}
|
||||
r := &ocds.Release{
|
||||
OCID: "ocds-x-1",
|
||||
ID: "001",
|
||||
Tag: []string{"tender"},
|
||||
Buyer: &ocds.OrganizationReference{ID: "GB-1", Name: "Council"},
|
||||
OCID: "ocds-x-1",
|
||||
ID: "001",
|
||||
Tag: []string{"tender"},
|
||||
Buyer: &ocds.OrganizationReference{
|
||||
ID: "GB-1",
|
||||
Name: "Council",
|
||||
},
|
||||
Tender: &ocds.Tender{
|
||||
Title: "Website",
|
||||
Status: "active",
|
||||
@@ -206,10 +279,17 @@ func TestConvertOne(t *testing.T) {
|
||||
Classification: &main,
|
||||
MainProcurementCategory: "services",
|
||||
ProcurementMethod: "open",
|
||||
Lots: []ocds.Lot{lotWith(&ocds.Suitability{SME: boolPtr(true)})},
|
||||
TenderPeriod: &ocds.Period{StartDate: &start, EndDate: &end},
|
||||
Value: &ocds.Value{Amount: &amount},
|
||||
Documents: []ocds.Document{{ID: "doc"}},
|
||||
Lots: []ocds.Lot{lotWith(&ocds.Suitability{
|
||||
SME: boolPtr(true),
|
||||
})},
|
||||
TenderPeriod: &ocds.Period{
|
||||
StartDate: &start,
|
||||
EndDate: &end,
|
||||
},
|
||||
Value: &ocds.Value{
|
||||
Amount: &amount,
|
||||
},
|
||||
Documents: []ocds.Document{{ID: "doc"}},
|
||||
},
|
||||
}
|
||||
|
||||
@@ -250,12 +330,20 @@ func TestConvertOneMinimal(t *testing.T) {
|
||||
}{
|
||||
{"nil tender", nil},
|
||||
{"empty tender", &ocds.Tender{}},
|
||||
{"value without amount", &ocds.Tender{Value: &ocds.Value{}}},
|
||||
{"empty tender period", &ocds.Tender{TenderPeriod: &ocds.Period{}}},
|
||||
{"value without amount", &ocds.Tender{
|
||||
Value: &ocds.Value{},
|
||||
}},
|
||||
{"empty tender period", &ocds.Tender{
|
||||
TenderPeriod: &ocds.Period{},
|
||||
}},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := convertOne(&ocds.Release{OCID: "ocds-x-2", ID: "1", Tender: tt.tender})
|
||||
got := convertOne(&ocds.Release{
|
||||
OCID: "ocds-x-2",
|
||||
ID: "1",
|
||||
Tender: tt.tender,
|
||||
})
|
||||
if got.UUID != "ocds-x-2" || got.BuyerId != "" || got.BuyerName != "" {
|
||||
t.Errorf("got %+v", got)
|
||||
}
|
||||
@@ -274,14 +362,22 @@ func TestConvert(t *testing.T) {
|
||||
t.Errorf("Convert(nil) = %v, want empty", got)
|
||||
}
|
||||
|
||||
day := func(d int) time.Time { return time.Date(2026, 1, d, 0, 0, 0, 0, time.UTC) }
|
||||
day := func(d int) time.Time {
|
||||
return time.Date(2026, 1, d, 0, 0, 0, 0, time.UTC)
|
||||
}
|
||||
releases := []ocds.Release{
|
||||
{OCID: "ocds-b", ID: "1", Date: day(1), Tender: &ocds.Tender{Title: "B"}},
|
||||
{OCID: "ocds-b", ID: "1", Date: day(1), Tender: &ocds.Tender{
|
||||
Title: "B",
|
||||
}},
|
||||
{OCID: "ocds-a", ID: "2", Date: day(3), Tender: &ocds.Tender{
|
||||
Title: "A latest", Lots: []ocds.Lot{{ID: "1"}, {ID: "2"}},
|
||||
}},
|
||||
{OCID: "ocds-a", ID: "1", Date: day(2), Tender: &ocds.Tender{Title: "A first"}},
|
||||
{OCID: "ocds-a", ID: "3", Date: day(1), Tender: &ocds.Tender{Title: "A oldest"}},
|
||||
{OCID: "ocds-a", ID: "1", Date: day(2), Tender: &ocds.Tender{
|
||||
Title: "A first",
|
||||
}},
|
||||
{OCID: "ocds-a", ID: "3", Date: day(1), Tender: &ocds.Tender{
|
||||
Title: "A oldest",
|
||||
}},
|
||||
}
|
||||
got := Convert(releases)
|
||||
|
||||
@@ -321,8 +417,12 @@ func TestConvert(t *testing.T) {
|
||||
func TestConvertSameDate(t *testing.T) {
|
||||
date := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)
|
||||
got := Convert([]ocds.Release{
|
||||
{OCID: "ocds-a", ID: "1", Date: date, Tender: &ocds.Tender{Title: "first"}},
|
||||
{OCID: "ocds-a", ID: "2", Date: date, Tender: &ocds.Tender{Title: "second"}},
|
||||
{OCID: "ocds-a", ID: "1", Date: date, Tender: &ocds.Tender{
|
||||
Title: "first",
|
||||
}},
|
||||
{OCID: "ocds-a", ID: "2", Date: date, Tender: &ocds.Tender{
|
||||
Title: "second",
|
||||
}},
|
||||
})
|
||||
if len(got) != 1 || got[0].Title != "second" {
|
||||
t.Fatalf("got %+v, want one opportunity titled %q", got, "second")
|
||||
@@ -335,8 +435,10 @@ func TestConvertSetsSource(t *testing.T) {
|
||||
const want = "FindATender"
|
||||
|
||||
for name, release := range map[string]*ocds.Release{
|
||||
"with tender": {OCID: "ocds-a", ID: "1", Tender: &ocds.Tender{Title: "A"}},
|
||||
"nil tender": {OCID: "ocds-b", ID: "1"},
|
||||
"with tender": {OCID: "ocds-a", ID: "1", Tender: &ocds.Tender{
|
||||
Title: "A",
|
||||
}},
|
||||
"nil tender": {OCID: "ocds-b", ID: "1"},
|
||||
} {
|
||||
if got := convertOne(release).Source; got != want {
|
||||
t.Errorf("convertOne (%s): Source = %q, want %q", name, got, want)
|
||||
@@ -344,8 +446,12 @@ func TestConvertSetsSource(t *testing.T) {
|
||||
}
|
||||
|
||||
for _, o := range Convert([]ocds.Release{
|
||||
{OCID: "ocds-a", ID: "1", Tender: &ocds.Tender{Title: "A"}},
|
||||
{OCID: "ocds-a", ID: "2", Tender: &ocds.Tender{Title: "A amended"}},
|
||||
{OCID: "ocds-a", ID: "1", Tender: &ocds.Tender{
|
||||
Title: "A",
|
||||
}},
|
||||
{OCID: "ocds-a", ID: "2", Tender: &ocds.Tender{
|
||||
Title: "A amended",
|
||||
}},
|
||||
{OCID: "ocds-b", ID: "1"},
|
||||
}) {
|
||||
if o.Source != want {
|
||||
|
||||
@@ -18,17 +18,17 @@ type Problem struct {
|
||||
|
||||
// WriteProblem writes a problem+json error response.
|
||||
func WriteProblem(w http.ResponseWriter, r *http.Request, status int, detail string) {
|
||||
p := Problem{
|
||||
problem := Problem{
|
||||
Type: "about:blank",
|
||||
Title: http.StatusText(status),
|
||||
Status: status,
|
||||
Detail: detail,
|
||||
}
|
||||
if r != nil {
|
||||
p.RequestID = RequestID(r.Context())
|
||||
problem.RequestID = RequestID(r.Context())
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/problem+json")
|
||||
w.Header().Set("X-Content-Type-Options", "nosniff")
|
||||
w.WriteHeader(status)
|
||||
json.NewEncoder(w).Encode(p) //nolint:errcheck,gosec // the client may have gone; nothing to do
|
||||
json.NewEncoder(w).Encode(problem) //nolint:errcheck,gosec // the client may have gone
|
||||
}
|
||||
@@ -26,32 +26,32 @@ func (r *statusRecorder) WriteHeader(code int) {
|
||||
r.ResponseWriter.WriteHeader(code)
|
||||
}
|
||||
|
||||
func (r *statusRecorder) Write(b []byte) (int, error) {
|
||||
func (r *statusRecorder) Write(data []byte) (int, error) {
|
||||
if r.status == 0 {
|
||||
r.status = http.StatusOK
|
||||
}
|
||||
n, err := r.ResponseWriter.Write(b)
|
||||
r.bytes += n
|
||||
return n, err
|
||||
written, err := r.ResponseWriter.Write(data)
|
||||
r.bytes += written
|
||||
return written, err
|
||||
}
|
||||
|
||||
// logField escapes a value for a quoted log field the way nginx does
|
||||
// (quotes, backslashes and control characters become \xHH) and uses "-"
|
||||
// for empty values.
|
||||
func logField(s string) string {
|
||||
if s == "" {
|
||||
func logField(value string) string {
|
||||
if value == "" {
|
||||
return "-"
|
||||
}
|
||||
var b strings.Builder
|
||||
for i := 0; i < len(s); i++ {
|
||||
c := s[i]
|
||||
if c == '"' || c == '\\' || c < 0x20 || c == 0x7f {
|
||||
fmt.Fprintf(&b, "\\x%02X", c)
|
||||
var builder strings.Builder
|
||||
for i := 0; i < len(value); i++ {
|
||||
char := value[i]
|
||||
if char == '"' || char == '\\' || char < 0x20 || char == 0x7f {
|
||||
fmt.Fprintf(&builder, "\\x%02X", char)
|
||||
} else {
|
||||
b.WriteByte(c)
|
||||
builder.WriteByte(char)
|
||||
}
|
||||
}
|
||||
return b.String()
|
||||
return builder.String()
|
||||
}
|
||||
|
||||
// clientKey attempts to produce a client host name as a key identifier to split
|
||||
@@ -71,26 +71,29 @@ func clientKey(req *http.Request) string {
|
||||
func AccessLog(logger *log.Logger, combined bool, next http.Handler) http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
|
||||
start := time.Now()
|
||||
rec := &statusRecorder{ResponseWriter: w}
|
||||
next.ServeHTTP(rec, req)
|
||||
if rec.status == 0 {
|
||||
rec.status = http.StatusOK
|
||||
recorder := &statusRecorder{
|
||||
ResponseWriter: w,
|
||||
}
|
||||
next.ServeHTTP(recorder, req)
|
||||
if recorder.status == 0 {
|
||||
recorder.status = http.StatusOK
|
||||
}
|
||||
|
||||
user := "-"
|
||||
if u, _, ok := req.BasicAuth(); ok && u != "" {
|
||||
user = logField(u)
|
||||
if username, _, ok := req.BasicAuth(); ok && username != "" {
|
||||
user = logField(username)
|
||||
}
|
||||
size := "-" // CLF uses "-" for an empty body
|
||||
if rec.bytes > 0 {
|
||||
size = strconv.Itoa(rec.bytes)
|
||||
if recorder.bytes > 0 {
|
||||
size = strconv.Itoa(recorder.bytes)
|
||||
}
|
||||
line := fmt.Sprintf(`%s - %s [%s] "%s" %d %s`,
|
||||
line := fmt.Sprintf(
|
||||
`%s - %s [%s] "%s" %d %s`,
|
||||
clientKey(req),
|
||||
user,
|
||||
start.Format("02/Jan/2006:15:04:05 -0700"),
|
||||
logField(req.Method+" "+req.RequestURI+" "+req.Proto),
|
||||
rec.status,
|
||||
recorder.status,
|
||||
size,
|
||||
)
|
||||
if combined {
|
||||
|
||||
@@ -40,7 +40,9 @@ func TestClientKey(t *testing.T) {
|
||||
{"", ""},
|
||||
}
|
||||
for _, tc := range tests {
|
||||
req := &http.Request{RemoteAddr: tc.remote}
|
||||
req := &http.Request{
|
||||
RemoteAddr: tc.remote,
|
||||
}
|
||||
if got := clientKey(req); got != tc.want {
|
||||
t.Errorf("clientKey(%q) = %q, want %q", tc.remote, got, tc.want)
|
||||
}
|
||||
@@ -49,7 +51,9 @@ func TestClientKey(t *testing.T) {
|
||||
|
||||
func TestStatusRecorder(t *testing.T) {
|
||||
t.Run("first WriteHeader wins", func(t *testing.T) {
|
||||
rec := &statusRecorder{ResponseWriter: httptest.NewRecorder()}
|
||||
rec := &statusRecorder{
|
||||
ResponseWriter: httptest.NewRecorder(),
|
||||
}
|
||||
rec.WriteHeader(http.StatusTeapot)
|
||||
rec.WriteHeader(http.StatusInternalServerError)
|
||||
if rec.status != http.StatusTeapot {
|
||||
@@ -58,7 +62,9 @@ func TestStatusRecorder(t *testing.T) {
|
||||
})
|
||||
t.Run("Write implies 200 and counts bytes", func(t *testing.T) {
|
||||
inner := httptest.NewRecorder()
|
||||
rec := &statusRecorder{ResponseWriter: inner}
|
||||
rec := &statusRecorder{
|
||||
ResponseWriter: inner,
|
||||
}
|
||||
rec.Write([]byte("hello"))
|
||||
rec.Write([]byte(" world"))
|
||||
if rec.status != http.StatusOK || rec.bytes != 11 {
|
||||
@@ -81,9 +87,10 @@ func TestAccessLog(t *testing.T) {
|
||||
want string // regexp for the whole line
|
||||
}{
|
||||
{
|
||||
name: "no body defaults to 200 and -",
|
||||
handler: func(http.ResponseWriter, *http.Request) {},
|
||||
want: `^192\.0\.2\.1 - - ` + clfTime + ` "GET /path\?q=1 HTTP/1\.1" 200 -$`,
|
||||
name: "no body defaults to 200 and -",
|
||||
handler: func(http.ResponseWriter, *http.Request) {
|
||||
},
|
||||
want: `^192\.0\.2\.1 - - ` + clfTime + ` "GET /path\?q=1 HTTP/1\.1" 200 -$`,
|
||||
},
|
||||
{
|
||||
name: "status and size",
|
||||
@@ -94,15 +101,20 @@ func TestAccessLog(t *testing.T) {
|
||||
want: `^192\.0\.2\.1 - - ` + clfTime + ` "GET /path\?q=1 HTTP/1\.1" 404 4$`,
|
||||
},
|
||||
{
|
||||
name: "basic auth user is escaped",
|
||||
handler: func(w http.ResponseWriter, _ *http.Request) { w.Write([]byte("x")) },
|
||||
setup: func(r *http.Request) { r.SetBasicAuth(`bo"b`, "pw") },
|
||||
want: `^192\.0\.2\.1 - bo\\x22b ` + clfTime + ` "GET /path\?q=1 HTTP/1\.1" 200 1$`,
|
||||
name: "basic auth user is escaped",
|
||||
handler: func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.Write([]byte("x"))
|
||||
},
|
||||
setup: func(r *http.Request) {
|
||||
r.SetBasicAuth(`bo"b`, "pw")
|
||||
},
|
||||
want: `^192\.0\.2\.1 - bo\\x22b ` + clfTime + ` "GET /path\?q=1 HTTP/1\.1" 200 1$`,
|
||||
},
|
||||
{
|
||||
name: "combined adds referer and user agent",
|
||||
combined: true,
|
||||
handler: func(http.ResponseWriter, *http.Request) {},
|
||||
handler: func(http.ResponseWriter, *http.Request) {
|
||||
},
|
||||
setup: func(r *http.Request) {
|
||||
r.Header.Set("Referer", "http://ref/")
|
||||
r.Header.Set("User-Agent", `ua "1"`)
|
||||
@@ -112,9 +124,12 @@ func TestAccessLog(t *testing.T) {
|
||||
{
|
||||
name: "combined with missing headers",
|
||||
combined: true,
|
||||
handler: func(http.ResponseWriter, *http.Request) {},
|
||||
setup: func(r *http.Request) { r.Header.Del("User-Agent") },
|
||||
want: `^192\.0\.2\.1 - - ` + clfTime + ` "GET /path\?q=1 HTTP/1\.1" 200 - "-" "-"$`,
|
||||
handler: func(http.ResponseWriter, *http.Request) {
|
||||
},
|
||||
setup: func(r *http.Request) {
|
||||
r.Header.Del("User-Agent")
|
||||
},
|
||||
want: `^192\.0\.2\.1 - - ` + clfTime + ` "GET /path\?q=1 HTTP/1\.1" 200 - "-" "-"$`,
|
||||
},
|
||||
}
|
||||
for _, tc := range tests {
|
||||
|
||||
@@ -135,9 +135,9 @@ func (h terminalHandler) WithGroup(name string) slog.Handler {
|
||||
}
|
||||
}
|
||||
|
||||
// IsJSONHandler reports whether h writes JSON, as the file logger does.
|
||||
func IsJSONHandler(h slog.Handler) bool {
|
||||
_, ok := h.(*slog.JSONHandler)
|
||||
// IsJSONHandler reports whether handler writes JSON, as the file logger does.
|
||||
func IsJSONHandler(handler slog.Handler) bool {
|
||||
_, ok := handler.(*slog.JSONHandler)
|
||||
return ok
|
||||
}
|
||||
|
||||
|
||||
@@ -67,9 +67,12 @@ func TestIsJSONHandler(t *testing.T) {
|
||||
{"tint", tint.NewTextHandler(io.Discard, nil), false},
|
||||
{"nil", nil, false},
|
||||
{"json with attrs", slog.NewJSONHandler(io.Discard, nil).WithAttrs(
|
||||
[]slog.Attr{slog.String("k", "v")}), true},
|
||||
[]slog.Attr{slog.String("k", "v")},
|
||||
), true},
|
||||
// Only slog's own JSON handler is recognised, not one wrapped in another.
|
||||
{"wrapped json", wrapper{slog.NewJSONHandler(io.Discard, nil)}, false},
|
||||
{"wrapped json", wrapper{
|
||||
Handler: slog.NewJSONHandler(io.Discard, nil),
|
||||
}, false},
|
||||
}
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
|
||||
@@ -24,6 +24,7 @@ type Limiter interface {
|
||||
// LimiterFunc adapts a function to the Limiter interface.
|
||||
type LimiterFunc func(key string) (bool, time.Duration)
|
||||
|
||||
// Allow implements Limiter by calling f.
|
||||
func (f LimiterFunc) Allow(key string) (bool, time.Duration) { return f(key) }
|
||||
|
||||
// KeyFunc derives the rate limit key from a request.
|
||||
@@ -34,22 +35,24 @@ func KeyByIP(r *http.Request) string { return ClientIP(r) }
|
||||
|
||||
// RateLimit turns a Limiter into middleware. Rejected requests get 429 with
|
||||
// a Retry-After header. A nil limiter yields a pass-through middleware.
|
||||
func RateLimit(l Limiter, key KeyFunc) func(http.Handler) http.Handler {
|
||||
if l == nil {
|
||||
return func(next http.Handler) http.Handler { return next }
|
||||
func RateLimit(limiter Limiter, key KeyFunc) func(http.Handler) http.Handler {
|
||||
if limiter == nil {
|
||||
return func(next http.Handler) http.Handler {
|
||||
return next
|
||||
}
|
||||
}
|
||||
if key == nil {
|
||||
key = KeyByIP
|
||||
}
|
||||
return func(next http.Handler) http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
ok, retry := l.Allow(key(r))
|
||||
ok, retry := limiter.Allow(key(r))
|
||||
if !ok {
|
||||
secs := int(math.Ceil(retry.Seconds()))
|
||||
if secs < 1 {
|
||||
secs = 1
|
||||
seconds := int(math.Ceil(retry.Seconds()))
|
||||
if seconds < 1 {
|
||||
seconds = 1
|
||||
}
|
||||
w.Header().Set("Retry-After", strconv.Itoa(secs))
|
||||
w.Header().Set("Retry-After", strconv.Itoa(seconds))
|
||||
httpx.WriteProblem(w, r, http.StatusTooManyRequests, "rate limit exceeded")
|
||||
return
|
||||
}
|
||||
@@ -74,22 +77,32 @@ type limiterShard struct {
|
||||
}
|
||||
|
||||
type limiterEntry struct {
|
||||
lim *rate.Limiter
|
||||
last time.Time
|
||||
limiter *rate.Limiter
|
||||
last time.Time
|
||||
}
|
||||
|
||||
// NewKeyedLimiter creates a limiter allowing rps sustained requests per
|
||||
// second with the given burst. A janitor goroutine evicts entries idle for
|
||||
// longer than ttl until ctx is cancelled.
|
||||
func NewKeyedLimiter(ctx context.Context, rps float64, burst int, ttl time.Duration) *KeyedLimiter {
|
||||
k := &KeyedLimiter{limit: rate.Limit(rps), burst: burst, ttl: ttl, seed: maphash.MakeSeed()}
|
||||
for i := range k.shards {
|
||||
k.shards[i].entries = make(map[string]*limiterEntry)
|
||||
// NewKeyedLimiter creates a limiter allowing requestsPerSecond sustained
|
||||
// requests per second with the given burst. A janitor goroutine evicts
|
||||
// entries idle for longer than ttl until ctx is cancelled.
|
||||
func NewKeyedLimiter(
|
||||
ctx context.Context,
|
||||
requestsPerSecond float64,
|
||||
burst int,
|
||||
ttl time.Duration,
|
||||
) *KeyedLimiter {
|
||||
limiter := &KeyedLimiter{
|
||||
limit: rate.Limit(requestsPerSecond),
|
||||
burst: burst,
|
||||
ttl: ttl,
|
||||
seed: maphash.MakeSeed(),
|
||||
}
|
||||
for i := range limiter.shards {
|
||||
limiter.shards[i].entries = make(map[string]*limiterEntry)
|
||||
}
|
||||
if ttl > 0 {
|
||||
go k.janitor(ctx)
|
||||
go limiter.janitor(ctx)
|
||||
}
|
||||
return k
|
||||
return limiter
|
||||
}
|
||||
|
||||
func (k *KeyedLimiter) shard(key string) *limiterShard {
|
||||
@@ -99,45 +112,47 @@ func (k *KeyedLimiter) shard(key string) *limiterShard {
|
||||
// Allow implements Limiter.
|
||||
func (k *KeyedLimiter) Allow(key string) (bool, time.Duration) {
|
||||
now := time.Now()
|
||||
s := k.shard(key)
|
||||
s.mu.Lock()
|
||||
e := s.entries[key]
|
||||
if e == nil {
|
||||
e = &limiterEntry{lim: rate.NewLimiter(k.limit, k.burst)}
|
||||
s.entries[key] = e
|
||||
shard := k.shard(key)
|
||||
shard.mu.Lock()
|
||||
entry := shard.entries[key]
|
||||
if entry == nil {
|
||||
entry = &limiterEntry{
|
||||
limiter: rate.NewLimiter(k.limit, k.burst),
|
||||
}
|
||||
shard.entries[key] = entry
|
||||
}
|
||||
e.last = now
|
||||
lim := e.lim
|
||||
s.mu.Unlock()
|
||||
entry.last = now
|
||||
limiter := entry.limiter
|
||||
shard.mu.Unlock()
|
||||
|
||||
res := lim.ReserveN(now, 1)
|
||||
res := limiter.ReserveN(now, 1)
|
||||
if !res.OK() {
|
||||
return false, time.Second
|
||||
}
|
||||
if d := res.DelayFrom(now); d > 0 {
|
||||
if delay := res.DelayFrom(now); delay > 0 {
|
||||
res.CancelAt(now)
|
||||
return false, d
|
||||
return false, delay
|
||||
}
|
||||
return true, 0
|
||||
}
|
||||
|
||||
func (k *KeyedLimiter) janitor(ctx context.Context) {
|
||||
t := time.NewTicker(k.ttl / 2)
|
||||
defer t.Stop()
|
||||
ticker := time.NewTicker(k.ttl / 2)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case now := <-t.C:
|
||||
case now := <-ticker.C:
|
||||
for i := range k.shards {
|
||||
s := &k.shards[i]
|
||||
s.mu.Lock()
|
||||
for key, e := range s.entries {
|
||||
if now.Sub(e.last) > k.ttl {
|
||||
delete(s.entries, key)
|
||||
shard := &k.shards[i]
|
||||
shard.mu.Lock()
|
||||
for key, entry := range shard.entries {
|
||||
if now.Sub(entry.last) > k.ttl {
|
||||
delete(shard.entries, key)
|
||||
}
|
||||
}
|
||||
s.mu.Unlock()
|
||||
shard.mu.Unlock()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -8,7 +8,8 @@ import (
|
||||
|
||||
func TestRateLimit(t *testing.T) {
|
||||
l := NewKeyedLimiter(t.Context(), 1, 2, 0)
|
||||
h := RateLimit(l, nil)(http.HandlerFunc(func(http.ResponseWriter, *http.Request) {}))
|
||||
h := RateLimit(l, nil)(http.HandlerFunc(func(http.ResponseWriter, *http.Request) {
|
||||
}))
|
||||
var last int
|
||||
for i := 0; i < 4; i++ {
|
||||
rec := httptest.NewRecorder()
|
||||
|
||||
@@ -20,8 +20,8 @@ type ClientInfo struct {
|
||||
// Client returns the ClientInfo stored by RealIP. Without RealIP it falls back
|
||||
// to the connection itself (see peerInfo) and ignores forwarding headers.
|
||||
func Client(r *http.Request) ClientInfo {
|
||||
if ci, ok := r.Context().Value(clientInfoKey).(ClientInfo); ok {
|
||||
return ci
|
||||
if info, ok := r.Context().Value(clientInfoKey).(ClientInfo); ok {
|
||||
return info
|
||||
}
|
||||
return peerInfo(r)
|
||||
}
|
||||
@@ -62,10 +62,14 @@ func peerInfo(r *http.Request) ClientInfo {
|
||||
|
||||
// RealIPOptions configures RealIP.
|
||||
type RealIPOptions struct {
|
||||
// Trusted are the proxies whose forwarding headers are believed.
|
||||
Trusted []netip.Prefix
|
||||
// Headers is processed in increasing priority order - the last header in the
|
||||
// list that is found is the header used to determine a client IP address.
|
||||
Headers []string // e.g. X-Forwarded-For, Forwarded, X-Real-IP
|
||||
// Headers are the headers the client IP is read from, in priority order:
|
||||
// the first of them that the request has and that gives an address is
|
||||
// used. E.g. X-Forwarded-For, Forwarded, X-Real-IP.
|
||||
Headers []string
|
||||
// StripUntrusted removes the forwarding headers from requests that are
|
||||
// not from a trusted proxy.
|
||||
StripUntrusted bool
|
||||
}
|
||||
|
||||
@@ -91,9 +95,9 @@ var forwardingHeaders = []string{
|
||||
// PROXY protocol is handled at the listener (see server package), so by the
|
||||
// time this runs r.RemoteAddr already holds the address from the PROXY header.
|
||||
func RealIP(opts RealIPOptions) func(http.Handler) http.Handler {
|
||||
trusted := func(a netip.Addr) bool {
|
||||
for _, p := range opts.Trusted {
|
||||
if p.Contains(a) {
|
||||
trusted := func(addr netip.Addr) bool {
|
||||
for _, prefix := range opts.Trusted {
|
||||
if prefix.Contains(addr) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
@@ -105,19 +109,19 @@ func RealIP(opts RealIPOptions) func(http.Handler) http.Handler {
|
||||
info := peerInfo(r)
|
||||
|
||||
if info.Peer.IsValid() && trusted(info.Peer) {
|
||||
for _, h := range opts.Headers {
|
||||
if ip, ok := clientFromHeader(r.Header, h, trusted); ok {
|
||||
for _, name := range opts.Headers {
|
||||
if ip, ok := clientFromHeader(r.Header, name, trusted); ok {
|
||||
info.IP = ip
|
||||
info.ViaProxy = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if p := forwardedProto(r.Header); p == "http" || p == "https" {
|
||||
info.Scheme = p
|
||||
if proto := forwardedProto(r.Header); proto == "http" || proto == "https" {
|
||||
info.Scheme = proto
|
||||
}
|
||||
} else if opts.StripUntrusted {
|
||||
for _, h := range forwardingHeaders {
|
||||
r.Header.Del(h)
|
||||
for _, name := range forwardingHeaders {
|
||||
r.Header.Del(name)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -128,17 +132,17 @@ func RealIP(opts RealIPOptions) func(http.Handler) http.Handler {
|
||||
}
|
||||
|
||||
func clientFromHeader(
|
||||
h http.Header, name string, trusted func(netip.Addr) bool,
|
||||
header http.Header, name string, trusted func(netip.Addr) bool,
|
||||
) (netip.Addr, bool) {
|
||||
switch http.CanonicalHeaderKey(name) {
|
||||
case "X-Forwarded-For":
|
||||
return walkChain(splitList(h.Values("X-Forwarded-For")), trusted)
|
||||
return walkChain(splitList(header.Values("X-Forwarded-For")), trusted)
|
||||
|
||||
case "Forwarded":
|
||||
var chain []string
|
||||
for _, el := range splitList(h.Values("Forwarded")) {
|
||||
if v, ok := forwardedParam(el, "for"); ok {
|
||||
chain = append(chain, v)
|
||||
for _, element := range splitList(header.Values("Forwarded")) {
|
||||
if value, ok := forwardedParam(element, "for"); ok {
|
||||
chain = append(chain, value)
|
||||
} else {
|
||||
chain = append(chain, "") // element without for= breaks the chain
|
||||
}
|
||||
@@ -146,8 +150,8 @@ func clientFromHeader(
|
||||
return walkChain(chain, trusted)
|
||||
|
||||
case "X-Real-Ip":
|
||||
if v := strings.TrimSpace(h.Get("X-Real-IP")); v != "" {
|
||||
return parseIPToken(v)
|
||||
if value := strings.TrimSpace(header.Get("X-Real-IP")); value != "" {
|
||||
return parseIPToken(value)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -174,15 +178,15 @@ func walkChain(
|
||||
return last, last.IsValid()
|
||||
}
|
||||
|
||||
func forwardedProto(h http.Header) string {
|
||||
if v := h.Get("X-Forwarded-Proto"); v != "" {
|
||||
first, _, _ := strings.Cut(v, ",")
|
||||
func forwardedProto(header http.Header) string {
|
||||
if value := header.Get("X-Forwarded-Proto"); value != "" {
|
||||
first, _, _ := strings.Cut(value, ",")
|
||||
return strings.ToLower(strings.TrimSpace(first))
|
||||
}
|
||||
|
||||
if els := splitList(h.Values("Forwarded")); len(els) > 0 {
|
||||
if v, ok := forwardedParam(els[0], "proto"); ok {
|
||||
return strings.ToLower(v)
|
||||
if elements := splitList(header.Values("Forwarded")); len(elements) > 0 {
|
||||
if value, ok := forwardedParam(elements[0], "proto"); ok {
|
||||
return strings.ToLower(value)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -193,9 +197,9 @@ func forwardedProto(h http.Header) string {
|
||||
// element (RFC 7239).
|
||||
func forwardedParam(element, key string) (string, bool) {
|
||||
for _, pair := range strings.Split(element, ";") {
|
||||
k, v, ok := strings.Cut(strings.TrimSpace(pair), "=")
|
||||
if ok && strings.EqualFold(k, key) {
|
||||
return strings.Trim(strings.TrimSpace(v), `"`), true
|
||||
name, value, ok := strings.Cut(strings.TrimSpace(pair), "=")
|
||||
if ok && strings.EqualFold(name, key) {
|
||||
return strings.Trim(strings.TrimSpace(value), `"`), true
|
||||
}
|
||||
}
|
||||
return "", false
|
||||
@@ -203,8 +207,8 @@ func forwardedParam(element, key string) (string, bool) {
|
||||
|
||||
func splitList(values []string) []string {
|
||||
var out []string
|
||||
for _, v := range values {
|
||||
for _, part := range strings.Split(v, ",") {
|
||||
for _, value := range values {
|
||||
for _, part := range strings.Split(value, ",") {
|
||||
if part = strings.TrimSpace(part); part != "" {
|
||||
out = append(out, part)
|
||||
}
|
||||
@@ -214,23 +218,23 @@ func splitList(values []string) []string {
|
||||
}
|
||||
|
||||
// parseIPToken accepts "1.2.3.4", "1.2.3.4:80", "::1", "[::1]" and "[::1]:80".
|
||||
func parseIPToken(s string) (netip.Addr, bool) {
|
||||
s = strings.Trim(strings.TrimSpace(s), `"`)
|
||||
if ap, err := netip.ParseAddrPort(s); err == nil {
|
||||
return ap.Addr().Unmap(), true
|
||||
func parseIPToken(token string) (netip.Addr, bool) {
|
||||
token = strings.Trim(strings.TrimSpace(token), `"`)
|
||||
if addrPort, err := netip.ParseAddrPort(token); err == nil {
|
||||
return addrPort.Addr().Unmap(), true
|
||||
}
|
||||
|
||||
s = strings.TrimSuffix(strings.TrimPrefix(s, "["), "]")
|
||||
if a, err := netip.ParseAddr(s); err == nil {
|
||||
return a.Unmap().WithZone(""), true
|
||||
token = strings.TrimSuffix(strings.TrimPrefix(token, "["), "]")
|
||||
if addr, err := netip.ParseAddr(token); err == nil {
|
||||
return addr.Unmap().WithZone(""), true
|
||||
}
|
||||
|
||||
return netip.Addr{}, false
|
||||
}
|
||||
|
||||
func addrFromRemote(remote string) netip.Addr {
|
||||
if a, ok := parseIPToken(remote); ok {
|
||||
return a
|
||||
if addr, ok := parseIPToken(remote); ok {
|
||||
return addr
|
||||
}
|
||||
return netip.Addr{}
|
||||
}
|
||||
@@ -12,7 +12,9 @@ import (
|
||||
func serveRealIP(t *testing.T, opts RealIPOptions, peer string, h map[string]string) ClientInfo {
|
||||
t.Helper()
|
||||
var got ClientInfo
|
||||
next := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { got = Client(r) })
|
||||
next := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
got = Client(r)
|
||||
})
|
||||
req := httptest.NewRequest("GET", "/", nil)
|
||||
req.RemoteAddr = peer
|
||||
for k, v := range h {
|
||||
|
||||
@@ -62,15 +62,15 @@ func LoadCatalog() (*Catalog, error) {
|
||||
}
|
||||
|
||||
// parseCatalog decodes and validates catalog JSON; name labels any error.
|
||||
func parseCatalog(b []byte, name string) (*Catalog, error) {
|
||||
var c Catalog
|
||||
if err := json.Unmarshal(b, &c); err != nil {
|
||||
func parseCatalog(data []byte, name string) (*Catalog, error) {
|
||||
var catalog Catalog
|
||||
if err := json.Unmarshal(data, &catalog); err != nil {
|
||||
return nil, fmt.Errorf("decoding %s: %w", name, err)
|
||||
}
|
||||
if err := c.validate(); err != nil {
|
||||
if err := catalog.validate(); err != nil {
|
||||
return nil, fmt.Errorf("invalid %s: %w", name, err)
|
||||
}
|
||||
return &c, nil
|
||||
return &catalog, nil
|
||||
}
|
||||
|
||||
// validate checks that every pool has at least one entry and that every
|
||||
@@ -83,11 +83,26 @@ func (c *Catalog) validate() error {
|
||||
name string
|
||||
n int
|
||||
}{
|
||||
{"buyers", len(c.Buyers)},
|
||||
{"suppliers", len(c.Suppliers)},
|
||||
{"subjects", len(c.Subjects)},
|
||||
{"methods", len(c.Methods)},
|
||||
{"classifications", len(c.Classifications)},
|
||||
{
|
||||
name: "buyers",
|
||||
n: len(c.Buyers),
|
||||
},
|
||||
{
|
||||
name: "suppliers",
|
||||
n: len(c.Suppliers),
|
||||
},
|
||||
{
|
||||
name: "subjects",
|
||||
n: len(c.Subjects),
|
||||
},
|
||||
{
|
||||
name: "methods",
|
||||
n: len(c.Methods),
|
||||
},
|
||||
{
|
||||
name: "classifications",
|
||||
n: len(c.Classifications),
|
||||
},
|
||||
} {
|
||||
if pool.n == 0 {
|
||||
errs = append(errs, fmt.Errorf("no %s", pool.name))
|
||||
@@ -96,25 +111,25 @@ func (c *Catalog) validate() error {
|
||||
|
||||
missing := map[string]bool{}
|
||||
check := func(keys ...string) {
|
||||
for _, k := range keys {
|
||||
if _, ok := c.Classifications[k]; k != "" && !ok {
|
||||
missing[k] = true
|
||||
for _, key := range keys {
|
||||
if _, ok := c.Classifications[key]; key != "" && !ok {
|
||||
missing[key] = true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for _, s := range c.Subjects {
|
||||
check(s.Classification)
|
||||
check(s.AdditionalClassifications...)
|
||||
for _, it := range s.Items {
|
||||
check(it.AdditionalClassifications...)
|
||||
for _, subject := range c.Subjects {
|
||||
check(subject.Classification)
|
||||
check(subject.AdditionalClassifications...)
|
||||
for _, item := range subject.Items {
|
||||
check(item.AdditionalClassifications...)
|
||||
}
|
||||
}
|
||||
|
||||
if len(missing) > 0 {
|
||||
keys := make([]string, 0, len(missing))
|
||||
for k := range missing {
|
||||
keys = append(keys, k)
|
||||
for key := range missing {
|
||||
keys = append(keys, key)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
if len(keys) > 5 {
|
||||
@@ -131,39 +146,39 @@ func (c *Catalog) validate() error {
|
||||
|
||||
// ---------- picking (always returns a copy the caller may modify) ----------
|
||||
|
||||
func (c *Catalog) buyer(r *rand.Rand) ocds.Organization {
|
||||
return clone(c.Buyers[r.Intn(len(c.Buyers))])
|
||||
func (c *Catalog) buyer(rng *rand.Rand) ocds.Organization {
|
||||
return clone(c.Buyers[rng.Intn(len(c.Buyers))])
|
||||
}
|
||||
|
||||
func (c *Catalog) supplier(r *rand.Rand) ocds.Organization {
|
||||
return clone(c.Suppliers[r.Intn(len(c.Suppliers))])
|
||||
func (c *Catalog) supplier(rng *rand.Rand) ocds.Organization {
|
||||
return clone(c.Suppliers[rng.Intn(len(c.Suppliers))])
|
||||
}
|
||||
|
||||
func (c *Catalog) method(r *rand.Rand) Method {
|
||||
return c.Methods[r.Intn(len(c.Methods))]
|
||||
func (c *Catalog) method(rng *rand.Rand) Method {
|
||||
return c.Methods[rng.Intn(len(c.Methods))]
|
||||
}
|
||||
|
||||
// subject picks a subject and returns it as an ocds.Tender, with its
|
||||
// classification keys resolved to full classifications.
|
||||
func (c *Catalog) subject(r *rand.Rand) ocds.Tender {
|
||||
s := clone(c.Subjects[r.Intn(len(c.Subjects))])
|
||||
t := s.Tender
|
||||
t.Classification = c.resolveOne(s.Classification)
|
||||
t.AdditionalClassifications = c.resolve(s.AdditionalClassifications)
|
||||
t.Items = nil
|
||||
for _, it := range s.Items {
|
||||
item := it.Item
|
||||
item.AdditionalClassifications = c.resolve(it.AdditionalClassifications)
|
||||
t.Items = append(t.Items, item)
|
||||
func (c *Catalog) subject(rng *rand.Rand) ocds.Tender {
|
||||
subject := clone(c.Subjects[rng.Intn(len(c.Subjects))])
|
||||
tender := subject.Tender
|
||||
tender.Classification = c.resolveOne(subject.Classification)
|
||||
tender.AdditionalClassifications = c.resolve(subject.AdditionalClassifications)
|
||||
tender.Items = nil
|
||||
for _, subjectItem := range subject.Items {
|
||||
item := subjectItem.Item
|
||||
item.AdditionalClassifications = c.resolve(subjectItem.AdditionalClassifications)
|
||||
tender.Items = append(tender.Items, item)
|
||||
}
|
||||
return t
|
||||
return tender
|
||||
}
|
||||
|
||||
// resolveOne returns a copy of the classification with key, or nil if there
|
||||
// is none.
|
||||
func (c *Catalog) resolveOne(key string) *ocds.Classification {
|
||||
if cl, ok := c.Classifications[key]; ok {
|
||||
return &cl
|
||||
if classification, ok := c.Classifications[key]; ok {
|
||||
return &classification
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -172,20 +187,20 @@ func (c *Catalog) resolveOne(key string) *ocds.Classification {
|
||||
// that have none.
|
||||
func (c *Catalog) resolve(keys []string) []ocds.Classification {
|
||||
var out []ocds.Classification
|
||||
for _, k := range keys {
|
||||
if cl, ok := c.Classifications[k]; ok {
|
||||
out = append(out, cl)
|
||||
for _, key := range keys {
|
||||
if classification, ok := c.Classifications[key]; ok {
|
||||
out = append(out, classification)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// clone deep-copies a value through JSON, which is all the generated types need.
|
||||
func clone[T any](v T) T {
|
||||
func clone[T any](value T) T {
|
||||
var out T
|
||||
b, err := json.Marshal(v)
|
||||
data, err := json.Marshal(value)
|
||||
if err == nil {
|
||||
json.Unmarshal(b, &out) //nolint:errcheck,gosec // b was just marshalled from a T
|
||||
json.Unmarshal(data, &out) //nolint:errcheck,gosec // b was just marshalled from a T
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -33,8 +33,13 @@ func TestEmbeddedCatalogIsValid(t *testing.T) {
|
||||
}
|
||||
if len(cat.Buyers) == 0 || len(cat.Suppliers) == 0 || len(cat.Subjects) == 0 ||
|
||||
len(cat.Methods) == 0 {
|
||||
t.Fatalf("embedded catalog has an empty pool: %d buyers, %d suppliers, %d subjects, %d methods",
|
||||
len(cat.Buyers), len(cat.Suppliers), len(cat.Subjects), len(cat.Methods))
|
||||
t.Fatalf(
|
||||
"embedded catalog has an empty pool: %d buyers, %d suppliers, %d subjects, %d methods",
|
||||
len(cat.Buyers),
|
||||
len(cat.Suppliers),
|
||||
len(cat.Subjects),
|
||||
len(cat.Methods),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -42,8 +47,13 @@ func TestParseCatalog(t *testing.T) {
|
||||
cat := loadTestCatalog(t)
|
||||
if len(cat.Buyers) != 2 || len(cat.Suppliers) != 1 || len(cat.Subjects) != 2 ||
|
||||
len(cat.Methods) != 2 {
|
||||
t.Fatalf("got %d buyers, %d suppliers, %d subjects, %d methods",
|
||||
len(cat.Buyers), len(cat.Suppliers), len(cat.Subjects), len(cat.Methods))
|
||||
t.Fatalf(
|
||||
"got %d buyers, %d suppliers, %d subjects, %d methods",
|
||||
len(cat.Buyers),
|
||||
len(cat.Suppliers),
|
||||
len(cat.Subjects),
|
||||
len(cat.Methods),
|
||||
)
|
||||
}
|
||||
if d := cat.Buyers[0].Details; d == nil || len(d.Classifications) != 1 ||
|
||||
d.Classifications[0].Scheme != "UK_CA_TYPE" {
|
||||
|
||||
@@ -6,8 +6,7 @@
|
||||
package mock
|
||||
|
||||
// Files kept in the mock-data directory.
|
||||
//
|
||||
// ReleasesFile is created and appended to by the mock.
|
||||
const (
|
||||
// ReleasesFile is created and appended to by the mock.
|
||||
ReleasesFile = "releases.jsonl"
|
||||
)
|
||||
@@ -23,12 +23,12 @@ type Generator struct {
|
||||
opts options
|
||||
}
|
||||
|
||||
// NewGenerator returns a generator drawing from cat and storing releases in
|
||||
// NewGenerator returns a generator drawing from catalog and storing releases in
|
||||
// store. The same seed and store contents give the same releases. It logs to
|
||||
// slog.Default() unless given WithLogger.
|
||||
func NewGenerator(cat *Catalog, store *Store, seed int64, opts ...Option) *Generator {
|
||||
func NewGenerator(catalog *Catalog, store *Store, seed int64, opts ...Option) *Generator {
|
||||
return &Generator{
|
||||
cat: cat,
|
||||
cat: catalog,
|
||||
store: store,
|
||||
rnd: rand.New(rand.NewSource(seed)), //nolint:gosec // seeded for repeatable mock data
|
||||
opts: applyOptions(opts),
|
||||
@@ -56,7 +56,7 @@ func (g *Generator) newProcedure(at time.Time) ocds.Release {
|
||||
buyer := g.cat.buyer(g.rnd)
|
||||
ensurePartyID(&buyer)
|
||||
buyer.Roles = []string{"buyer"}
|
||||
m := g.cat.method(g.rnd)
|
||||
method := g.cat.method(g.rnd)
|
||||
|
||||
tag, status := "tender", "active"
|
||||
if g.rnd.Float64() < 0.2 {
|
||||
@@ -64,38 +64,38 @@ func (g *Generator) newProcedure(at time.Time) ocds.Release {
|
||||
}
|
||||
ocid := g.newOCID()
|
||||
|
||||
t := g.cat.subject(g.rnd)
|
||||
t.ID = ocds.ID(ocid)
|
||||
t.ProcurementMethod = m.Method
|
||||
t.ProcurementMethodDetails = m.Details
|
||||
tender := g.cat.subject(g.rnd)
|
||||
tender.ID = ocds.ID(ocid)
|
||||
tender.ProcurementMethod = method.Method
|
||||
tender.ProcurementMethodDetails = method.Details
|
||||
|
||||
factor := 0.5 + g.rnd.Float64()*1.5
|
||||
if t.Value == nil {
|
||||
if tender.Value == nil {
|
||||
amount := float64((g.rnd.Intn(5000) + 10) * 1000)
|
||||
t.Value = &ocds.Value{
|
||||
tender.Value = &ocds.Value{
|
||||
Amount: &amount,
|
||||
Currency: "GBP",
|
||||
}
|
||||
} else {
|
||||
scaleValue(t.Value, factor)
|
||||
scaleValue(tender.Value, factor)
|
||||
}
|
||||
|
||||
for i := range t.Lots {
|
||||
scaleValue(t.Lots[i].Value, factor)
|
||||
for i := range tender.Lots {
|
||||
scaleValue(tender.Lots[i].Value, factor)
|
||||
}
|
||||
|
||||
if len(t.Lots) == 0 {
|
||||
v := clone(*t.Value)
|
||||
t.Lots = []ocds.Lot{{
|
||||
if len(tender.Lots) == 0 {
|
||||
value := clone(*tender.Value)
|
||||
tender.Lots = []ocds.Lot{{
|
||||
ID: "1",
|
||||
Title: t.Title,
|
||||
Value: &v,
|
||||
Title: tender.Title,
|
||||
Value: &value,
|
||||
}}
|
||||
}
|
||||
|
||||
setStatus(&t, status)
|
||||
setStatus(&tender, status)
|
||||
if status == "active" {
|
||||
t.TenderPeriod = &ocds.Period{
|
||||
tender.TenderPeriod = &ocds.Period{
|
||||
EndDate: ptr(at.AddDate(0, 0, 30+g.rnd.Intn(30))),
|
||||
}
|
||||
}
|
||||
@@ -111,7 +111,7 @@ func (g *Generator) newProcedure(at time.Time) ocds.Release {
|
||||
ID: ocds.ID(buyer.ID),
|
||||
Name: buyer.Name,
|
||||
},
|
||||
Tender: &t,
|
||||
Tender: &tender,
|
||||
Language: "en",
|
||||
}
|
||||
}
|
||||
@@ -120,13 +120,13 @@ func (g *Generator) newProcedure(at time.Time) ocds.Release {
|
||||
// tender, and a tender either has its deadline extended (tenderUpdate) or is
|
||||
// awarded to a supplier from the catalog.
|
||||
func (g *Generator) advance(prev ocds.Release, at time.Time) ocds.Release {
|
||||
r := clone(prev)
|
||||
r.ID = g.store.NextNoticeID(at.Year())
|
||||
r.Date = at
|
||||
r.Awards = nil
|
||||
if r.Tender == nil {
|
||||
r.Tender = &ocds.Tender{
|
||||
ID: ocds.ID(r.OCID),
|
||||
release := clone(prev)
|
||||
release.ID = g.store.NextNoticeID(at.Year())
|
||||
release.Date = at
|
||||
release.Awards = nil
|
||||
if release.Tender == nil {
|
||||
release.Tender = &ocds.Tender{
|
||||
ID: ocds.ID(release.OCID),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -139,60 +139,60 @@ func (g *Generator) advance(prev ocds.Release, at time.Time) ocds.Release {
|
||||
next = "tenderUpdate"
|
||||
}
|
||||
}
|
||||
r.Tag = []string{next}
|
||||
release.Tag = []string{next}
|
||||
|
||||
switch next {
|
||||
case "tender":
|
||||
setStatus(r.Tender, "active")
|
||||
r.Tender.TenderPeriod = &ocds.Period{
|
||||
setStatus(release.Tender, "active")
|
||||
release.Tender.TenderPeriod = &ocds.Period{
|
||||
EndDate: ptr(at.AddDate(0, 0, 30+g.rnd.Intn(30))),
|
||||
}
|
||||
case "tenderUpdate":
|
||||
// A typical update: the deadline is extended by two weeks.
|
||||
end := at.AddDate(0, 0, 30)
|
||||
if tp := r.Tender.TenderPeriod; tp != nil && tp.EndDate != nil {
|
||||
end = *tp.EndDate
|
||||
if period := release.Tender.TenderPeriod; period != nil && period.EndDate != nil {
|
||||
end = *period.EndDate
|
||||
}
|
||||
r.Tender.TenderPeriod = &ocds.Period{
|
||||
release.Tender.TenderPeriod = &ocds.Period{
|
||||
EndDate: ptr(end.AddDate(0, 0, 14)),
|
||||
}
|
||||
case "award":
|
||||
setStatus(r.Tender, "complete")
|
||||
sup := g.cat.supplier(g.rnd)
|
||||
ensurePartyID(&sup)
|
||||
sup.Roles = []string{"supplier"}
|
||||
r.Parties = append(r.Parties, sup)
|
||||
setStatus(release.Tender, "complete")
|
||||
supplier := g.cat.supplier(g.rnd)
|
||||
ensurePartyID(&supplier)
|
||||
supplier.Roles = []string{"supplier"}
|
||||
release.Parties = append(release.Parties, supplier)
|
||||
|
||||
var value *ocds.Value
|
||||
if r.Tender.Value != nil {
|
||||
v := clone(*r.Tender.Value)
|
||||
scaleValue(&v, 0.8+g.rnd.Float64()*0.2)
|
||||
value = &v
|
||||
if release.Tender.Value != nil {
|
||||
scaled := clone(*release.Tender.Value)
|
||||
scaleValue(&scaled, 0.8+g.rnd.Float64()*0.2)
|
||||
value = &scaled
|
||||
}
|
||||
|
||||
var lotIDs []string
|
||||
for _, lot := range r.Tender.Lots {
|
||||
for _, lot := range release.Tender.Lots {
|
||||
if lot.ID != "" {
|
||||
lotIDs = append(lotIDs, lot.ID)
|
||||
}
|
||||
}
|
||||
|
||||
r.Awards = []ocds.Award{{
|
||||
release.Awards = []ocds.Award{{
|
||||
ID: "1",
|
||||
Status: "active",
|
||||
Date: ptr(at),
|
||||
Value: value,
|
||||
Suppliers: []ocds.OrganizationReference{
|
||||
{
|
||||
ID: ocds.ID(sup.ID),
|
||||
Name: sup.Name,
|
||||
ID: ocds.ID(supplier.ID),
|
||||
Name: supplier.Name,
|
||||
},
|
||||
},
|
||||
RelatedLots: lotIDs,
|
||||
}}
|
||||
}
|
||||
|
||||
return r
|
||||
return release
|
||||
}
|
||||
|
||||
// newOCID returns a random ocid not already in the store.
|
||||
@@ -224,25 +224,25 @@ func (g *Generator) CatchUp(
|
||||
start = last
|
||||
}
|
||||
|
||||
n := 0
|
||||
for t := start.Add(g.gap(min, max)); !t.After(now); t = t.Add(g.gap(min, max)) {
|
||||
if err := g.store.AppendNoSync(g.Make(t)); err != nil {
|
||||
return n, err
|
||||
count := 0
|
||||
for when := start.Add(g.gap(min, max)); !when.After(now); when = when.Add(g.gap(min, max)) {
|
||||
if err := g.store.AppendNoSync(g.Make(when)); err != nil {
|
||||
return count, err
|
||||
}
|
||||
n++
|
||||
count++
|
||||
}
|
||||
|
||||
return n, g.store.Sync()
|
||||
return count, g.store.Sync()
|
||||
}
|
||||
|
||||
// Run publishes a new release after a random gap between min and max until ctx ends.
|
||||
func (g *Generator) Run(ctx context.Context, min, max time.Duration) {
|
||||
logger := g.opts.logger("generator")
|
||||
for {
|
||||
d := g.gap(min, max)
|
||||
logger.Info("next release", "in", d.Round(time.Second))
|
||||
gap := g.gap(min, max)
|
||||
logger.Info("next release", "in", gap.Round(time.Second))
|
||||
|
||||
timer := time.NewTimer(d)
|
||||
timer := time.NewTimer(gap)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
timer.Stop()
|
||||
@@ -250,46 +250,51 @@ func (g *Generator) Run(ctx context.Context, min, max time.Duration) {
|
||||
case <-timer.C:
|
||||
}
|
||||
|
||||
r := g.Make(time.Now())
|
||||
if err := g.store.Append(r); err != nil {
|
||||
release := g.Make(time.Now())
|
||||
if err := g.store.Append(release); err != nil {
|
||||
logger.Error("storing release failed", "error", err)
|
||||
continue
|
||||
}
|
||||
|
||||
logger.Info("published release", "release_id", r.ID, "tag", r.Tag[0], "ocid", r.OCID)
|
||||
logger.Info(
|
||||
"published release",
|
||||
"release_id", release.ID,
|
||||
"tag", release.Tag[0],
|
||||
"ocid", release.OCID,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
// ---------- helpers ----------
|
||||
|
||||
func ptr[T any](v T) *T { return &v }
|
||||
func ptr[T any](value T) *T { return &value }
|
||||
|
||||
func setStatus(t *ocds.Tender, status string) {
|
||||
t.Status = status
|
||||
for i := range t.Lots {
|
||||
t.Lots[i].Status = status
|
||||
func setStatus(tender *ocds.Tender, status string) {
|
||||
tender.Status = status
|
||||
for i := range tender.Lots {
|
||||
tender.Lots[i].Status = status
|
||||
}
|
||||
}
|
||||
|
||||
// scaleValue multiplies v's amounts by f, rounded to the nearest 100.
|
||||
func scaleValue(v *ocds.Value, f float64) {
|
||||
if v == nil {
|
||||
// scaleValue multiplies value's amounts by factor, rounded to the nearest 100.
|
||||
func scaleValue(value *ocds.Value, factor float64) {
|
||||
if value == nil {
|
||||
return
|
||||
}
|
||||
for _, a := range []**float64{&v.Amount, &v.AmountGross} {
|
||||
if *a != nil {
|
||||
*a = ptr(math.Round(**a*f/100) * 100)
|
||||
for _, amount := range []**float64{&value.Amount, &value.AmountGross} {
|
||||
if *amount != nil {
|
||||
*amount = ptr(math.Round(**amount*factor/100) * 100)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ensurePartyID gives p an id derived from its name if it has none, so the
|
||||
// ensurePartyID gives party an id derived from its name if it has none, so the
|
||||
// same organisation always gets the same id.
|
||||
func ensurePartyID(p *ocds.Organization) {
|
||||
if p.ID != "" {
|
||||
func ensurePartyID(party *ocds.Organization) {
|
||||
if party.ID != "" {
|
||||
return
|
||||
}
|
||||
h := fnv.New32a()
|
||||
h.Write([]byte(p.Name)) //nolint:gosec // never fails
|
||||
p.ID = fmt.Sprintf("GB-FTS-%d", h.Sum32()%1000000)
|
||||
hasher := fnv.New32a()
|
||||
hasher.Write([]byte(party.Name)) //nolint:gosec // never fails
|
||||
party.ID = fmt.Sprintf("GB-FTS-%d", hasher.Sum32()%1000000)
|
||||
}
|
||||
@@ -17,12 +17,16 @@ func quietLog(t *testing.T) {
|
||||
t.Helper()
|
||||
w := log.Writer()
|
||||
log.SetOutput(io.Discard)
|
||||
t.Cleanup(func() { log.SetOutput(w) })
|
||||
t.Cleanup(func() {
|
||||
log.SetOutput(w)
|
||||
})
|
||||
}
|
||||
|
||||
func TestGap(t *testing.T) {
|
||||
//nolint:gosec // G404: seeded math/rand keeps the test deterministic
|
||||
g := &Generator{rnd: rand.New(rand.NewSource(1))}
|
||||
g := &Generator{
|
||||
rnd: rand.New(rand.NewSource(1)),
|
||||
}
|
||||
if d := g.gap(time.Second, time.Second); d != time.Second {
|
||||
t.Errorf("min == max: %s", d)
|
||||
}
|
||||
@@ -39,7 +43,10 @@ func TestGap(t *testing.T) {
|
||||
func TestScaleValue(t *testing.T) {
|
||||
scaleValue(nil, 2) // must not panic
|
||||
|
||||
v := &ocds.Value{Amount: ptr(12345.0), Currency: "GBP"}
|
||||
v := &ocds.Value{
|
||||
Amount: ptr(12345.0),
|
||||
Currency: "GBP",
|
||||
}
|
||||
scaleValue(v, 1.5)
|
||||
if *v.Amount != 18500 { // 18517.5 rounded to the nearest 100
|
||||
t.Errorf("amount = %v, want 18500", *v.Amount)
|
||||
@@ -48,7 +55,10 @@ func TestScaleValue(t *testing.T) {
|
||||
t.Error("nil gross amount was set")
|
||||
}
|
||||
|
||||
v = &ocds.Value{Amount: ptr(1000.0), AmountGross: ptr(1200.0)}
|
||||
v = &ocds.Value{
|
||||
Amount: ptr(1000.0),
|
||||
AmountGross: ptr(1200.0),
|
||||
}
|
||||
scaleValue(v, 0.5)
|
||||
if *v.Amount != 500 || *v.AmountGross != 600 {
|
||||
t.Errorf("amounts = %v / %v, want 500 / 600", *v.Amount, *v.AmountGross)
|
||||
@@ -56,15 +66,24 @@ func TestScaleValue(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestEnsurePartyID(t *testing.T) {
|
||||
p := ocds.Organization{ID: "GB-EXISTING", Name: "Council"}
|
||||
p := ocds.Organization{
|
||||
ID: "GB-EXISTING",
|
||||
Name: "Council",
|
||||
}
|
||||
ensurePartyID(&p)
|
||||
if p.ID != "GB-EXISTING" {
|
||||
t.Errorf("existing id replaced: %s", p.ID)
|
||||
}
|
||||
|
||||
a := ocds.Organization{Name: "Somewhere Council"}
|
||||
b := ocds.Organization{Name: "Somewhere Council"}
|
||||
c := ocds.Organization{Name: "Elsewhere Council"}
|
||||
a := ocds.Organization{
|
||||
Name: "Somewhere Council",
|
||||
}
|
||||
b := ocds.Organization{
|
||||
Name: "Somewhere Council",
|
||||
}
|
||||
c := ocds.Organization{
|
||||
Name: "Elsewhere Council",
|
||||
}
|
||||
ensurePartyID(&a)
|
||||
ensurePartyID(&b)
|
||||
ensurePartyID(&c)
|
||||
@@ -90,7 +109,13 @@ func TestAdvance(t *testing.T) {
|
||||
at := time.Date(2026, 10, 8, 12, 0, 0, 0, time.UTC)
|
||||
|
||||
// A planning notice without a tender becomes an active tender.
|
||||
r := g.advance(ocds.Release{OCID: "ocds-x", Tag: []string{"planning"}}, at)
|
||||
r := g.advance(
|
||||
ocds.Release{
|
||||
OCID: "ocds-x",
|
||||
Tag: []string{"planning"},
|
||||
},
|
||||
at,
|
||||
)
|
||||
if r.Tag[0] != "tender" || r.Tender == nil || r.Tender.Status != "active" ||
|
||||
r.Tender.TenderPeriod == nil || r.Tender.TenderPeriod.EndDate == nil {
|
||||
t.Fatalf("planning -> %v, tender %+v", r.Tag, r.Tender)
|
||||
@@ -105,10 +130,15 @@ func TestAdvance(t *testing.T) {
|
||||
OCID: "ocds-y",
|
||||
Tag: []string{"tender"},
|
||||
Tender: &ocds.Tender{
|
||||
ID: "ocds-y",
|
||||
Value: &ocds.Value{Amount: ptr(10000.0), Currency: "GBP"},
|
||||
Lots: []ocds.Lot{{ID: "1"}, {ID: ""}},
|
||||
TenderPeriod: &ocds.Period{EndDate: &end},
|
||||
ID: "ocds-y",
|
||||
Value: &ocds.Value{
|
||||
Amount: ptr(10000.0),
|
||||
Currency: "GBP",
|
||||
},
|
||||
Lots: []ocds.Lot{{ID: "1"}, {ID: ""}},
|
||||
TenderPeriod: &ocds.Period{
|
||||
EndDate: &end,
|
||||
},
|
||||
},
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
@@ -147,7 +177,13 @@ func TestAdvance(t *testing.T) {
|
||||
|
||||
// An update without a deadline gets 30 + 14 days; a tender with no value
|
||||
// is awarded with no value.
|
||||
prev = ocds.Release{OCID: "ocds-z", Tag: []string{"tender"}, Tender: &ocds.Tender{ID: "ocds-z"}}
|
||||
prev = ocds.Release{
|
||||
OCID: "ocds-z",
|
||||
Tag: []string{"tender"},
|
||||
Tender: &ocds.Tender{
|
||||
ID: "ocds-z",
|
||||
},
|
||||
}
|
||||
for i := 0; i < 50; i++ {
|
||||
r := g.advance(prev, at)
|
||||
switch r.Tag[0] {
|
||||
|
||||
@@ -14,11 +14,17 @@ import (
|
||||
"git.avsdev.uk/Go/job-search/internal/mock/findatender/ocds"
|
||||
)
|
||||
|
||||
// Paging and request parameters of the release package endpoint.
|
||||
const (
|
||||
// defaultLimit is the page size when the request sets no limit.
|
||||
defaultLimit = 100
|
||||
maxLimit = 100
|
||||
timeLayout = "2006-01-02T15:04:05"
|
||||
BasePath = "/api/1.0/ocdsReleasePackages"
|
||||
// maxLimit is the largest page size a request may ask for.
|
||||
maxLimit = 100
|
||||
// timeLayout is the documented layout of updatedFrom and updatedTo; see
|
||||
// parseTime for the others accepted.
|
||||
timeLayout = "2006-01-02T15:04:05"
|
||||
// BasePath is the path the endpoint is served at.
|
||||
BasePath = "/api/1.0/ocdsReleasePackages"
|
||||
)
|
||||
|
||||
// server serves release packages from a Store; see NewHandler.
|
||||
@@ -36,10 +42,18 @@ type server struct {
|
||||
// Stored release dates are UTC; comparisons are on absolute instants. It
|
||||
// logs to slog.Default() unless given WithLogger.
|
||||
func NewHandler(
|
||||
store *Store, baseURL string, loc *time.Location, opts ...Option,
|
||||
store *Store,
|
||||
baseURL string,
|
||||
location *time.Location,
|
||||
opts ...Option,
|
||||
) http.HandlerFunc {
|
||||
s := &server{store: store, baseURL: baseURL, loc: loc, opts: applyOptions(opts)}
|
||||
return s.handle
|
||||
handler := &server{
|
||||
store: store,
|
||||
baseURL: baseURL,
|
||||
loc: location,
|
||||
opts: applyOptions(opts),
|
||||
}
|
||||
return handler.handle
|
||||
}
|
||||
|
||||
// encodeCursor makes the opaque cursor for a next link that resumes at store
|
||||
@@ -50,18 +64,18 @@ func encodeCursor(i int) string {
|
||||
|
||||
// decodeCursor returns the release index in a cursor made by encodeCursor.
|
||||
// Anything else, including a negative index, is an error.
|
||||
func decodeCursor(c string) (int, error) {
|
||||
b, err := base64.RawURLEncoding.DecodeString(c)
|
||||
if err != nil || !strings.HasPrefix(string(b), "idx:") {
|
||||
func decodeCursor(cursor string) (int, error) {
|
||||
decoded, err := base64.RawURLEncoding.DecodeString(cursor)
|
||||
if err != nil || !strings.HasPrefix(string(decoded), "idx:") {
|
||||
return 0, fmt.Errorf("invalid cursor")
|
||||
}
|
||||
|
||||
n, err := strconv.Atoi(strings.TrimPrefix(string(b), "idx:"))
|
||||
if err != nil || n < 0 {
|
||||
index, err := strconv.Atoi(strings.TrimPrefix(string(decoded), "idx:"))
|
||||
if err != nil || index < 0 {
|
||||
return 0, fmt.Errorf("invalid cursor")
|
||||
}
|
||||
|
||||
return n, nil
|
||||
return index, nil
|
||||
}
|
||||
|
||||
func badRequest(w http.ResponseWriter, msg string) {
|
||||
@@ -72,20 +86,20 @@ func badRequest(w http.ResponseWriter, msg string) {
|
||||
}
|
||||
|
||||
// parseTime reads a query timestamp. Values without an offset (the documented
|
||||
// yyyy-MM-ddTHH:mm:ss, or a plain date) are local time in loc; values with an
|
||||
// yyyy-MM-ddTHH:mm:ss, or a plain date) are local time in location; values with an
|
||||
// offset or Z (RFC 3339) mean exactly what they say.
|
||||
func parseTime(v string, loc *time.Location) (time.Time, error) {
|
||||
if loc == nil {
|
||||
loc = time.UTC
|
||||
func parseTime(value string, location *time.Location) (time.Time, error) {
|
||||
if location == nil {
|
||||
location = time.UTC
|
||||
}
|
||||
|
||||
if t, err := time.Parse(time.RFC3339, v); err == nil {
|
||||
return t, nil
|
||||
if parsed, err := time.Parse(time.RFC3339, value); err == nil {
|
||||
return parsed, nil
|
||||
}
|
||||
|
||||
for _, layout := range []string{timeLayout, "2006-01-02"} {
|
||||
if t, err := time.ParseInLocation(layout, v, loc); err == nil {
|
||||
return t, nil
|
||||
if parsed, err := time.ParseInLocation(layout, value, location); err == nil {
|
||||
return parsed, nil
|
||||
}
|
||||
}
|
||||
|
||||
@@ -107,15 +121,15 @@ func (s *server) handle(w http.ResponseWriter, req *http.Request) {
|
||||
return
|
||||
}
|
||||
|
||||
q := req.URL.Query()
|
||||
pq, err := s.parseQuery(q)
|
||||
query := req.URL.Query()
|
||||
page, err := s.parseQuery(query)
|
||||
if err != nil {
|
||||
badRequest(w, err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
releases, next, more := s.store.Page(pq.from, pq.upper, pq.start, pq.limit)
|
||||
pkg := ocds.ReleasePackage{
|
||||
releases, next, more := s.store.Page(page.from, page.upper, page.start, page.limit)
|
||||
releasePackage := ocds.ReleasePackage{
|
||||
URI: s.baseURL + req.URL.RequestURI(),
|
||||
Version: "1.1",
|
||||
Extensions: ocds.DeclaredExtensions,
|
||||
@@ -127,73 +141,78 @@ func (s *server) handle(w http.ResponseWriter, req *http.Request) {
|
||||
Releases: releases,
|
||||
}
|
||||
if more {
|
||||
pkg.Links = &ocds.ReleasePackageLinks{Next: s.nextLink(q, next)}
|
||||
releasePackage.Links = &ocds.ReleasePackageLinks{
|
||||
Next: s.nextLink(query, next),
|
||||
}
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "application/json; charset=utf-8")
|
||||
w.Header().Set("Access-Control-Allow-Origin", "*")
|
||||
enc := json.NewEncoder(w)
|
||||
enc.SetIndent("", " ")
|
||||
enc.SetEscapeHTML(false)
|
||||
if err := enc.Encode(pkg); err != nil {
|
||||
encoder := json.NewEncoder(w)
|
||||
encoder.SetIndent("", " ")
|
||||
encoder.SetEscapeHTML(false)
|
||||
if err := encoder.Encode(releasePackage); err != nil {
|
||||
s.opts.logger("handler").Warn("writing response failed", "error", err)
|
||||
}
|
||||
}
|
||||
|
||||
// parseQuery reads and checks the query parameters. Its errors are meant for
|
||||
// the client.
|
||||
func (s *server) parseQuery(q url.Values) (pageQuery, error) {
|
||||
pq := pageQuery{limit: defaultLimit, upper: time.Now().UTC()}
|
||||
|
||||
if v := q.Get("limit"); v != "" {
|
||||
n, err := strconv.Atoi(v)
|
||||
if err != nil || n < 1 || n > maxLimit {
|
||||
return pq, fmt.Errorf("limit must be an integer between 1 and %d", maxLimit)
|
||||
}
|
||||
pq.limit = n
|
||||
func (s *server) parseQuery(query url.Values) (pageQuery, error) {
|
||||
page := pageQuery{
|
||||
limit: defaultLimit,
|
||||
upper: time.Now().UTC(),
|
||||
}
|
||||
|
||||
if v := q.Get("updatedFrom"); v != "" {
|
||||
t, err := parseTime(v, s.loc)
|
||||
if value := query.Get("limit"); value != "" {
|
||||
number, err := strconv.Atoi(value)
|
||||
if err != nil || number < 1 || number > maxLimit {
|
||||
return page, fmt.Errorf("limit must be an integer between 1 and %d", maxLimit)
|
||||
}
|
||||
page.limit = number
|
||||
}
|
||||
|
||||
if value := query.Get("updatedFrom"); value != "" {
|
||||
from, err := parseTime(value, s.loc)
|
||||
if err != nil {
|
||||
return pq, fmt.Errorf("updatedFrom %w", err)
|
||||
return page, fmt.Errorf("updatedFrom %w", err)
|
||||
}
|
||||
pq.from = t
|
||||
page.from = from
|
||||
}
|
||||
|
||||
if v := q.Get("updatedTo"); v != "" {
|
||||
t, err := parseTime(v, s.loc)
|
||||
if value := query.Get("updatedTo"); value != "" {
|
||||
to, err := parseTime(value, s.loc)
|
||||
if err != nil {
|
||||
return pq, fmt.Errorf("updatedTo %w", err)
|
||||
return page, fmt.Errorf("updatedTo %w", err)
|
||||
}
|
||||
if q.Get("updatedFrom") != "" && t.Before(pq.from) {
|
||||
return pq, errors.New("updatedTo must not be before updatedFrom")
|
||||
if query.Get("updatedFrom") != "" && to.Before(page.from) {
|
||||
return page, errors.New("updatedTo must not be before updatedFrom")
|
||||
}
|
||||
if t.Before(pq.upper) {
|
||||
pq.upper = t
|
||||
if to.Before(page.upper) {
|
||||
page.upper = to
|
||||
}
|
||||
}
|
||||
|
||||
if c := q.Get("cursor"); c != "" {
|
||||
n, err := decodeCursor(c)
|
||||
if cursor := query.Get("cursor"); cursor != "" {
|
||||
index, err := decodeCursor(cursor)
|
||||
if err != nil {
|
||||
return pq, err
|
||||
return page, err
|
||||
}
|
||||
pq.start = n
|
||||
page.start = index
|
||||
}
|
||||
|
||||
return pq, nil
|
||||
return page, nil
|
||||
}
|
||||
|
||||
// nextLink is the URL of the page starting at release index next: the same
|
||||
// query with a new cursor.
|
||||
func (s *server) nextLink(q url.Values, next int) string {
|
||||
nq := url.Values{}
|
||||
for k, vs := range q {
|
||||
if k != "cursor" {
|
||||
nq[k] = vs
|
||||
func (s *server) nextLink(query url.Values, next int) string {
|
||||
nextQuery := url.Values{}
|
||||
for key, values := range query {
|
||||
if key != "cursor" {
|
||||
nextQuery[key] = values
|
||||
}
|
||||
}
|
||||
nq.Set("cursor", encodeCursor(next))
|
||||
return s.baseURL + BasePath + "?" + nq.Encode()
|
||||
nextQuery.Set("cursor", encodeCursor(next))
|
||||
return s.baseURL + BasePath + "?" + nextQuery.Encode()
|
||||
}
|
||||
@@ -79,7 +79,9 @@ func openTestStore(t *testing.T) *Store {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { store.Close() })
|
||||
t.Cleanup(func() {
|
||||
store.Close()
|
||||
})
|
||||
return store
|
||||
}
|
||||
|
||||
@@ -92,7 +94,9 @@ func TestHandlerRejectsBadRequests(t *testing.T) {
|
||||
t.Errorf("POST: status %d, want 405", rec.Code)
|
||||
}
|
||||
|
||||
cursor := func(s string) string { return base64.RawURLEncoding.EncodeToString([]byte(s)) }
|
||||
cursor := func(s string) string {
|
||||
return base64.RawURLEncoding.EncodeToString([]byte(s))
|
||||
}
|
||||
cases := map[string]string{
|
||||
"limit=0": "limit must be an integer between 1 and 100",
|
||||
"limit=101": "limit must be an integer between 1 and 100",
|
||||
|
||||
@@ -50,46 +50,50 @@ func (rl *RateLimiter) allow(key string) (bool, int) {
|
||||
now := rl.now().UTC()
|
||||
hour := now.Truncate(time.Hour)
|
||||
|
||||
c, ok := rl.clients[key]
|
||||
if !ok || !c.hour.Equal(hour) {
|
||||
client, ok := rl.clients[key]
|
||||
if !ok || !client.hour.Equal(hour) {
|
||||
// First request of this clock hour: it anchors the windows.
|
||||
if !ok && len(rl.clients) > 10000 {
|
||||
rl.prune(hour)
|
||||
}
|
||||
c = &clientWindow{hour: hour, anchor: now, windowStart: now}
|
||||
rl.clients[key] = c
|
||||
client = &clientWindow{
|
||||
hour: hour,
|
||||
anchor: now,
|
||||
windowStart: now,
|
||||
}
|
||||
rl.clients[key] = client
|
||||
}
|
||||
|
||||
// Move to whichever fixed window, counted from the anchor, now falls in.
|
||||
idx := now.Sub(c.anchor) / rl.window
|
||||
if ws := c.anchor.Add(idx * rl.window); !ws.Equal(c.windowStart) {
|
||||
c.windowStart = ws
|
||||
c.count = 0
|
||||
index := now.Sub(client.anchor) / rl.window
|
||||
if windowStart := client.anchor.Add(index * rl.window); !windowStart.Equal(client.windowStart) {
|
||||
client.windowStart = windowStart
|
||||
client.count = 0
|
||||
}
|
||||
|
||||
if c.count < rl.limit {
|
||||
c.count++
|
||||
if client.count < rl.limit {
|
||||
client.count++
|
||||
return true, 0
|
||||
}
|
||||
|
||||
// The next window opens at the end of this one, or at the top of the
|
||||
// next hour if that comes first (because a new hour re-anchors).
|
||||
next := c.windowStart.Add(rl.window)
|
||||
if nh := hour.Add(time.Hour); nh.Before(next) {
|
||||
next = nh
|
||||
next := client.windowStart.Add(rl.window)
|
||||
if nextHour := hour.Add(time.Hour); nextHour.Before(next) {
|
||||
next = nextHour
|
||||
}
|
||||
secs := int((next.Sub(now) + time.Second - 1) / time.Second) // round up
|
||||
if secs < 1 {
|
||||
secs = 1
|
||||
seconds := int((next.Sub(now) + time.Second - 1) / time.Second) // round up
|
||||
if seconds < 1 {
|
||||
seconds = 1
|
||||
}
|
||||
return false, secs
|
||||
return false, seconds
|
||||
}
|
||||
|
||||
// prune drops clients whose state belongs to an earlier hour.
|
||||
func (rl *RateLimiter) prune(hour time.Time) {
|
||||
for k, c := range rl.clients {
|
||||
if c.hour.Before(hour) {
|
||||
delete(rl.clients, k)
|
||||
for key, client := range rl.clients {
|
||||
if client.hour.Before(hour) {
|
||||
delete(rl.clients, key)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -12,7 +12,9 @@ import (
|
||||
func TestRateLimitWindows(t *testing.T) {
|
||||
now := time.Date(2026, 10, 6, 10, 3, 20, 0, time.UTC) // first request of the hour
|
||||
rl := NewRateLimiter(12, 5*time.Minute)
|
||||
rl.now = func() time.Time { return now }
|
||||
rl.now = func() time.Time {
|
||||
return now
|
||||
}
|
||||
|
||||
for i := 0; i < 12; i++ {
|
||||
if ok, _ := rl.allow("a"); !ok {
|
||||
@@ -63,7 +65,9 @@ func TestClientKey(t *testing.T) {
|
||||
func TestRateLimitMiddleware(t *testing.T) {
|
||||
now := time.Date(2026, 10, 6, 10, 0, 0, 0, time.UTC)
|
||||
rl := NewRateLimiter(2, 5*time.Minute)
|
||||
rl.now = func() time.Time { return now }
|
||||
rl.now = func() time.Time {
|
||||
return now
|
||||
}
|
||||
|
||||
calls := 0
|
||||
h := rl.Middleware(func(w http.ResponseWriter, req *http.Request) {
|
||||
@@ -108,7 +112,9 @@ func TestRateLimitMiddleware(t *testing.T) {
|
||||
func TestRateLimitPrunesStaleClients(t *testing.T) {
|
||||
now := time.Date(2026, 10, 6, 10, 30, 0, 0, time.UTC)
|
||||
rl := NewRateLimiter(1, time.Minute)
|
||||
rl.now = func() time.Time { return now }
|
||||
rl.now = func() time.Time {
|
||||
return now
|
||||
}
|
||||
|
||||
for i := 0; i <= 10000; i++ {
|
||||
rl.allow(strconv.Itoa(i))
|
||||
|
||||
@@ -43,7 +43,7 @@ type Store struct {
|
||||
// left incomplete by a crash is ended so new ones start on a fresh line.
|
||||
// It logs to slog.Default() unless given WithLogger.
|
||||
func OpenStore(path string, opts ...Option) (*Store, error) {
|
||||
s := &Store{
|
||||
store := &Store{
|
||||
latest: map[string]int{},
|
||||
lastNum: map[int]int{},
|
||||
openPos: map[string]int{},
|
||||
@@ -53,57 +53,57 @@ func OpenStore(path string, opts ...Option) (*Store, error) {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
releases, err := readReleases(s.opts.logger("store"), path)
|
||||
releases, err := readReleases(store.opts.logger("store"), path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// The file is written in order, but sort anyway in case it was hand-edited.
|
||||
sort.SliceStable(releases, func(i, j int) bool {
|
||||
return releases[i].Date.Before(releases[j].Date)
|
||||
sort.SliceStable(releases, func(i, other int) bool {
|
||||
return releases[i].Date.Before(releases[other].Date)
|
||||
})
|
||||
for _, r := range releases {
|
||||
s.releases = append(s.releases, r)
|
||||
s.times = append(s.times, r.Date)
|
||||
for _, release := range releases {
|
||||
store.releases = append(store.releases, release)
|
||||
store.times = append(store.times, release.Date)
|
||||
}
|
||||
for i, r := range s.releases {
|
||||
s.index(i, r)
|
||||
for i, release := range store.releases {
|
||||
store.index(i, release)
|
||||
}
|
||||
|
||||
if s.f, err = openForAppend(path); err != nil {
|
||||
if store.f, err = openForAppend(path); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return s, nil
|
||||
return store, nil
|
||||
}
|
||||
|
||||
// readReleases reads the releases stored at path, skipping blank lines and
|
||||
// lines that can't be decoded (which are logged to logger). A missing file
|
||||
// holds no releases.
|
||||
func readReleases(logger *slog.Logger, path string) ([]ocds.Release, error) {
|
||||
in, err := os.Open(filepath.Clean(path))
|
||||
input, err := os.Open(filepath.Clean(path))
|
||||
if os.IsNotExist(err) {
|
||||
return nil, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer in.Close() //nolint:errcheck // read only
|
||||
defer input.Close() //nolint:errcheck // read only
|
||||
|
||||
var releases []ocds.Release
|
||||
sc := bufio.NewScanner(in)
|
||||
sc.Buffer(make([]byte, 1<<20), 64<<20)
|
||||
for line := 1; sc.Scan(); line++ {
|
||||
b := bytes.TrimSpace(sc.Bytes())
|
||||
if len(b) == 0 {
|
||||
scanner := bufio.NewScanner(input)
|
||||
scanner.Buffer(make([]byte, 1<<20), 64<<20)
|
||||
for line := 1; scanner.Scan(); line++ {
|
||||
data := bytes.TrimSpace(scanner.Bytes())
|
||||
if len(data) == 0 {
|
||||
continue
|
||||
}
|
||||
var r ocds.Release
|
||||
if err := json.Unmarshal(b, &r); err != nil {
|
||||
var release ocds.Release
|
||||
if err := json.Unmarshal(data, &release); err != nil {
|
||||
logger.Warn("skipping unreadable line", "line", line, "error", err)
|
||||
continue
|
||||
}
|
||||
releases = append(releases, r)
|
||||
releases = append(releases, release)
|
||||
}
|
||||
if err := sc.Err(); err != nil {
|
||||
if err := scanner.Err(); err != nil {
|
||||
return nil, fmt.Errorf("reading %s: %w", path, err)
|
||||
}
|
||||
return releases, nil
|
||||
@@ -113,60 +113,60 @@ func readReleases(logger *slog.Logger, path string) ([]ocds.Release, error) {
|
||||
// mid-write, it ends the partial record so the next one starts on a fresh
|
||||
// line.
|
||||
func openForAppend(path string) (*os.File, error) {
|
||||
f, err := os.OpenFile(filepath.Clean(path), os.O_CREATE|os.O_RDWR|os.O_APPEND, 0o600)
|
||||
file, err := os.OpenFile(filepath.Clean(path), os.O_CREATE|os.O_RDWR|os.O_APPEND, 0o600)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if st, err := f.Stat(); err == nil && st.Size() > 0 {
|
||||
if info, err := file.Stat(); err == nil && info.Size() > 0 {
|
||||
last := make([]byte, 1)
|
||||
if _, err := f.ReadAt(last, st.Size()-1); err == nil && last[0] != '\n' {
|
||||
if _, err := f.Write([]byte("\n")); err != nil {
|
||||
return nil, errors.Join(fmt.Errorf("repairing %s: %w", path, err), f.Close())
|
||||
if _, err := file.ReadAt(last, info.Size()-1); err == nil && last[0] != '\n' {
|
||||
if _, err := file.Write([]byte("\n")); err != nil {
|
||||
return nil, errors.Join(fmt.Errorf("repairing %s: %w", path, err), file.Close())
|
||||
}
|
||||
}
|
||||
}
|
||||
return f, nil
|
||||
return file, nil
|
||||
}
|
||||
|
||||
// index records release r, at position i, in the lookups: the ocid's latest
|
||||
// index records release, at position i, in the lookups: the ocid's latest
|
||||
// release, whether its procedure is still open (an award closes it), and the
|
||||
// highest notice number used each year.
|
||||
func (s *Store) index(i int, r ocds.Release) {
|
||||
s.latest[r.OCID] = i
|
||||
if len(r.Tag) > 0 && r.Tag[0] == "award" {
|
||||
s.closeProcedure(r.OCID)
|
||||
} else if _, ok := s.openPos[r.OCID]; !ok {
|
||||
s.openPos[r.OCID] = len(s.open)
|
||||
s.open = append(s.open, r.OCID)
|
||||
func (s *Store) index(i int, release ocds.Release) {
|
||||
s.latest[release.OCID] = i
|
||||
if len(release.Tag) > 0 && release.Tag[0] == "award" {
|
||||
s.closeProcedure(release.OCID)
|
||||
} else if _, ok := s.openPos[release.OCID]; !ok {
|
||||
s.openPos[release.OCID] = len(s.open)
|
||||
s.open = append(s.open, release.OCID)
|
||||
}
|
||||
if num, year, ok := strings.Cut(r.ID, "-"); ok {
|
||||
n, err1 := strconv.Atoi(num)
|
||||
y, err2 := strconv.Atoi(year)
|
||||
if err1 == nil && err2 == nil && n > s.lastNum[y] {
|
||||
s.lastNum[y] = n
|
||||
if numberText, year, ok := strings.Cut(release.ID, "-"); ok {
|
||||
number, numberErr := strconv.Atoi(numberText)
|
||||
yearNumber, yearErr := strconv.Atoi(year)
|
||||
if numberErr == nil && yearErr == nil && number > s.lastNum[yearNumber] {
|
||||
s.lastNum[yearNumber] = number
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// closeProcedure removes ocid from the open procedures, if it is there.
|
||||
func (s *Store) closeProcedure(ocid string) {
|
||||
pos, ok := s.openPos[ocid]
|
||||
position, ok := s.openPos[ocid]
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
last := len(s.open) - 1
|
||||
s.open[pos] = s.open[last]
|
||||
s.openPos[s.open[pos]] = pos
|
||||
s.open[position] = s.open[last]
|
||||
s.openPos[s.open[position]] = position
|
||||
s.open = s.open[:last]
|
||||
delete(s.openPos, ocid)
|
||||
}
|
||||
|
||||
// Append persists a release and flushes it to disk.
|
||||
func (s *Store) Append(r ocds.Release) error { return s.append(r, true) }
|
||||
func (s *Store) Append(release ocds.Release) error { return s.append(release, true) }
|
||||
|
||||
// AppendNoSync persists a release without an fsync; call Sync afterwards.
|
||||
// Used for bulk catch-up, where a sync per record would be very slow.
|
||||
func (s *Store) AppendNoSync(r ocds.Release) error { return s.append(r, false) }
|
||||
func (s *Store) AppendNoSync(release ocds.Release) error { return s.append(release, false) }
|
||||
|
||||
// Sync flushes appended releases to disk. Call it after AppendNoSync.
|
||||
func (s *Store) Sync() error {
|
||||
@@ -175,22 +175,22 @@ func (s *Store) Sync() error {
|
||||
return s.f.Sync()
|
||||
}
|
||||
|
||||
// append stores r, syncing the file to disk if sync is true. A date earlier
|
||||
// append stores release, syncing the file to disk if sync is true. A date earlier
|
||||
// than the newest stored release (for example after a clock change) is moved
|
||||
// up so the order stays valid.
|
||||
func (s *Store) append(r ocds.Release, sync bool) error {
|
||||
func (s *Store) append(release ocds.Release, sync bool) error {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
|
||||
r.Date = r.Date.UTC()
|
||||
if n := len(s.times); n > 0 && r.Date.Before(s.times[n-1]) {
|
||||
r.Date = s.times[n-1]
|
||||
release.Date = release.Date.UTC()
|
||||
if count := len(s.times); count > 0 && release.Date.Before(s.times[count-1]) {
|
||||
release.Date = s.times[count-1]
|
||||
}
|
||||
b, err := json.Marshal(r)
|
||||
data, err := json.Marshal(release)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := s.f.Write(append(b, '\n')); err != nil {
|
||||
if _, err := s.f.Write(append(data, '\n')); err != nil {
|
||||
return err
|
||||
}
|
||||
if sync {
|
||||
@@ -198,9 +198,9 @@ func (s *Store) append(r ocds.Release, sync bool) error {
|
||||
return err
|
||||
}
|
||||
}
|
||||
s.releases = append(s.releases, r)
|
||||
s.times = append(s.times, r.Date)
|
||||
s.index(len(s.releases)-1, r)
|
||||
s.releases = append(s.releases, release)
|
||||
s.times = append(s.times, release.Date)
|
||||
s.index(len(s.releases)-1, release)
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -210,7 +210,9 @@ func (s *Store) Page(from, upper time.Time, start, limit int) ([]ocds.Release, i
|
||||
s.mu.RLock()
|
||||
defer s.mu.RUnlock()
|
||||
|
||||
i := sort.Search(len(s.times), func(i int) bool { return !s.times[i].Before(from) })
|
||||
i := sort.Search(len(s.times), func(i int) bool {
|
||||
return !s.times[i].Before(from)
|
||||
})
|
||||
if start > i {
|
||||
i = start
|
||||
}
|
||||
@@ -259,11 +261,11 @@ func (s *Store) NextNoticeID(year int) string {
|
||||
// RandomOpen picks a procedure that has not been awarded and whose latest
|
||||
// release is after since. Procedures idle since before then are dropped from
|
||||
// the open set, as if they had been abandoned.
|
||||
func (s *Store) RandomOpen(r *rand.Rand, since time.Time) (ocds.Release, bool) {
|
||||
func (s *Store) RandomOpen(rng *rand.Rand, since time.Time) (ocds.Release, bool) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
for len(s.open) > 0 {
|
||||
ocid := s.open[r.Intn(len(s.open))]
|
||||
ocid := s.open[rng.Intn(len(s.open))]
|
||||
i := s.latest[ocid]
|
||||
if s.times[i].After(since) {
|
||||
return s.releases[i], true
|
||||
|
||||
@@ -65,7 +65,11 @@ func TestStorePersistsAndGeneratorProgresses(t *testing.T) {
|
||||
}
|
||||
|
||||
// The HTTP handler pages through the store without gaps or repeats.
|
||||
srv := &server{store: store2, baseURL: "", loc: time.UTC}
|
||||
srv := &server{
|
||||
store: store2,
|
||||
baseURL: "",
|
||||
loc: time.UTC,
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
next := BasePath + "?updatedFrom=2026-10-07T00:00:00&limit=7"
|
||||
for pages := 0; next != ""; pages++ {
|
||||
@@ -105,7 +109,12 @@ func writeLines(t *testing.T, path string, lines ...string) {
|
||||
|
||||
func releaseLine(t *testing.T, ocid, id, tag string, at time.Time) string {
|
||||
t.Helper()
|
||||
b, err := json.Marshal(ocds.Release{OCID: ocid, ID: id, Date: at, Tag: []string{tag}})
|
||||
b, err := json.Marshal(ocds.Release{
|
||||
OCID: ocid,
|
||||
ID: id,
|
||||
Date: at,
|
||||
Tag: []string{tag},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -117,7 +126,9 @@ func releaseLine(t *testing.T, ocid, id, tag string, at time.Time) string {
|
||||
func TestOpenStoreRecoversFromMessyFile(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "r.jsonl")
|
||||
t0 := time.Date(2026, 10, 8, 9, 0, 0, 0, time.UTC)
|
||||
writeLines(t, path,
|
||||
writeLines(
|
||||
t,
|
||||
path,
|
||||
releaseLine(t, "ocds-b", "000007-2026", "tender", t0.Add(2*time.Hour)),
|
||||
"{not json",
|
||||
"",
|
||||
@@ -225,9 +236,19 @@ func TestAppendKeepsDateOrder(t *testing.T) {
|
||||
}
|
||||
defer store.Close()
|
||||
t0 := time.Date(2026, 10, 8, 9, 0, 0, 0, time.UTC)
|
||||
store.Append(ocds.Release{OCID: "a", ID: "000001-2026", Date: t0, Tag: []string{"tender"}})
|
||||
store.Append(ocds.Release{
|
||||
OCID: "a",
|
||||
ID: "000001-2026",
|
||||
Date: t0,
|
||||
Tag: []string{"tender"},
|
||||
})
|
||||
store.Append(
|
||||
ocds.Release{OCID: "b", ID: "000002-2026", Date: t0.Add(-time.Hour), Tag: []string{"tender"}},
|
||||
ocds.Release{
|
||||
OCID: "b",
|
||||
ID: "000002-2026",
|
||||
Date: t0.Add(-time.Hour),
|
||||
Tag: []string{"tender"},
|
||||
},
|
||||
)
|
||||
if last, _ := store.Last(); !last.Equal(t0) {
|
||||
t.Errorf("Last = %s, want %s", last, t0)
|
||||
@@ -235,7 +256,11 @@ func TestAppendKeepsDateOrder(t *testing.T) {
|
||||
|
||||
// Appending to a closed store fails and leaves the store unchanged.
|
||||
store.Close()
|
||||
if err := store.Append(ocds.Release{OCID: "c", Date: t0, Tag: []string{"tender"}}); err == nil {
|
||||
if err := store.Append(ocds.Release{
|
||||
OCID: "c",
|
||||
Date: t0,
|
||||
Tag: []string{"tender"},
|
||||
}); err == nil {
|
||||
t.Error("append after close: expected error")
|
||||
}
|
||||
if store.Len() != 2 || store.HasOCID("c") {
|
||||
@@ -251,9 +276,19 @@ func TestRandomOpenDropsStaleProcedures(t *testing.T) {
|
||||
}
|
||||
defer store.Close()
|
||||
t0 := time.Date(2026, 10, 8, 9, 0, 0, 0, time.UTC)
|
||||
store.Append(ocds.Release{OCID: "old", ID: "000001-2026", Date: t0, Tag: []string{"tender"}})
|
||||
store.Append(ocds.Release{
|
||||
OCID: "old",
|
||||
ID: "000001-2026",
|
||||
Date: t0,
|
||||
Tag: []string{"tender"},
|
||||
})
|
||||
store.Append(
|
||||
ocds.Release{OCID: "new", ID: "000002-2026", Date: t0.Add(time.Hour), Tag: []string{"tender"}},
|
||||
ocds.Release{
|
||||
OCID: "new",
|
||||
ID: "000002-2026",
|
||||
Date: t0.Add(time.Hour),
|
||||
Tag: []string{"tender"},
|
||||
},
|
||||
)
|
||||
|
||||
//nolint:gosec // G404: seeded math/rand keeps the test deterministic
|
||||
|
||||
@@ -24,7 +24,9 @@ func OpenDB(path string) (*sql.DB, error) {
|
||||
// ErrNeedsMigration. Intended for tests and throwaway databases; applications
|
||||
// should migrate explicitly with the migrate command.
|
||||
func WithAutoMigrate() Option {
|
||||
return func(o *options) { o.autoMigrate = true }
|
||||
return func(opts *options) {
|
||||
opts.autoMigrate = true
|
||||
}
|
||||
}
|
||||
|
||||
// dsn builds the mattn/go-sqlite3 connection string.
|
||||
|
||||
+45
-35
@@ -46,8 +46,8 @@ func NewMigrator(db *sql.DB) (*goose.Provider, error) {
|
||||
}
|
||||
|
||||
// LatestVersion is the highest migration version in this build.
|
||||
func LatestVersion(p *goose.Provider) int64 {
|
||||
sources := p.ListSources() // sorted ascending
|
||||
func LatestVersion(provider *goose.Provider) int64 {
|
||||
sources := provider.ListSources() // sorted ascending
|
||||
if len(sources) == 0 {
|
||||
return 0
|
||||
}
|
||||
@@ -56,26 +56,34 @@ func LatestVersion(p *goose.Provider) int64 {
|
||||
|
||||
// checkVersion returns ErrNeedsMigration or ErrSchemaTooNew unless the
|
||||
// database is exactly at this build's schema.
|
||||
func checkVersion(ctx context.Context, p *goose.Provider) error {
|
||||
current, err := p.GetDBVersion(ctx)
|
||||
func checkVersion(ctx context.Context, provider *goose.Provider) error {
|
||||
current, err := provider.GetDBVersion(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
latest := LatestVersion(p)
|
||||
latest := LatestVersion(provider)
|
||||
if current > latest {
|
||||
return fmt.Errorf("%w (database is at version %d, this build knows up to %d)",
|
||||
ErrSchemaTooNew, current, latest)
|
||||
return fmt.Errorf(
|
||||
"%w (database is at version %d, this build knows up to %d)",
|
||||
ErrSchemaTooNew,
|
||||
current,
|
||||
latest,
|
||||
)
|
||||
}
|
||||
|
||||
pending, err := p.HasPending(ctx)
|
||||
pending, err := provider.HasPending(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if pending {
|
||||
return fmt.Errorf("%w (database is at version %d, latest is %d)",
|
||||
ErrNeedsMigration, current, latest)
|
||||
return fmt.Errorf(
|
||||
"%w (database is at version %d, latest is %d)",
|
||||
ErrNeedsMigration,
|
||||
current,
|
||||
latest,
|
||||
)
|
||||
}
|
||||
|
||||
return nil
|
||||
@@ -83,53 +91,55 @@ func checkVersion(ctx context.Context, p *goose.Provider) error {
|
||||
|
||||
// checkNotNewer refuses to work on a database migrated by a newer build:
|
||||
// this build has neither the newer migrations nor their down steps.
|
||||
func checkNotNewer(ctx context.Context, m *goose.Provider) error {
|
||||
current, err := m.GetDBVersion(ctx)
|
||||
func checkNotNewer(ctx context.Context, provider *goose.Provider) error {
|
||||
current, err := provider.GetDBVersion(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if latest := LatestVersion(m); current > latest {
|
||||
if latest := LatestVersion(provider); current > latest {
|
||||
return fmt.Errorf(
|
||||
"database is at version %d but this build only knows up to %d; use a newer build",
|
||||
current, latest)
|
||||
current,
|
||||
latest,
|
||||
)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// prepareSchema migrates (with WithAutoMigrate) or verifies the schema
|
||||
// version.
|
||||
func prepareSchema(ctx context.Context, db *sql.DB, o options) error {
|
||||
p, err := NewMigrator(db)
|
||||
func prepareSchema(ctx context.Context, db *sql.DB, opts options) error {
|
||||
provider, err := NewMigrator(db)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if o.autoMigrate {
|
||||
if _, err := p.Up(ctx); err != nil {
|
||||
if opts.autoMigrate {
|
||||
if _, err := provider.Up(ctx); err != nil {
|
||||
return fmt.Errorf("store: migrating database: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
return checkVersion(ctx, p)
|
||||
return checkVersion(ctx, provider)
|
||||
}
|
||||
|
||||
// StepUp applies steps pending migrations, or all of them when steps is 0.
|
||||
func StepUp(
|
||||
ctx context.Context, m *goose.Provider, steps int,
|
||||
ctx context.Context, provider *goose.Provider, steps int,
|
||||
) ([]*goose.MigrationResult, error) {
|
||||
var results []*goose.MigrationResult
|
||||
|
||||
if err := checkNotNewer(ctx, m); err != nil {
|
||||
if err := checkNotNewer(ctx, provider); err != nil {
|
||||
return results, err
|
||||
}
|
||||
|
||||
var err error
|
||||
if steps == 0 {
|
||||
results, err = m.Up(ctx)
|
||||
results, err = provider.Up(ctx)
|
||||
} else {
|
||||
for range steps {
|
||||
var r *goose.MigrationResult
|
||||
r, err = m.UpByOne(ctx)
|
||||
var result *goose.MigrationResult
|
||||
result, err = provider.UpByOne(ctx)
|
||||
if errors.Is(err, goose.ErrNoNextVersion) {
|
||||
err = nil
|
||||
break
|
||||
@@ -137,7 +147,7 @@ func StepUp(
|
||||
if err != nil {
|
||||
break
|
||||
}
|
||||
results = append(results, r)
|
||||
results = append(results, result)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -148,15 +158,15 @@ func StepUp(
|
||||
// is 0. Rolling everything back deletes all data, so confirmDrop is checked
|
||||
// first in that case.
|
||||
func StepDown(
|
||||
ctx context.Context, m *goose.Provider, steps int, confirmDrop bool,
|
||||
ctx context.Context, provider *goose.Provider, steps int, confirmDrop bool,
|
||||
) ([]*goose.MigrationResult, error) {
|
||||
var results []*goose.MigrationResult
|
||||
|
||||
if err := checkNotNewer(ctx, m); err != nil {
|
||||
if err := checkNotNewer(ctx, provider); err != nil {
|
||||
return results, err
|
||||
}
|
||||
|
||||
current, err := m.GetDBVersion(ctx)
|
||||
current, err := provider.GetDBVersion(ctx)
|
||||
if err != nil {
|
||||
return results, err
|
||||
}
|
||||
@@ -164,13 +174,13 @@ func StepDown(
|
||||
return results, nil
|
||||
}
|
||||
|
||||
statuses, err := m.Status(ctx)
|
||||
statuses, err := provider.Status(ctx)
|
||||
if err != nil {
|
||||
return results, err
|
||||
}
|
||||
applied := 0
|
||||
for _, s := range statuses {
|
||||
if s.State == goose.StateApplied {
|
||||
for _, status := range statuses {
|
||||
if status.State == goose.StateApplied {
|
||||
applied++
|
||||
}
|
||||
}
|
||||
@@ -180,11 +190,11 @@ func StepDown(
|
||||
}
|
||||
|
||||
if all {
|
||||
results, err = m.DownTo(ctx, 0)
|
||||
results, err = provider.DownTo(ctx, 0)
|
||||
} else {
|
||||
for range steps {
|
||||
var r *goose.MigrationResult
|
||||
r, err = m.Down(ctx)
|
||||
var result *goose.MigrationResult
|
||||
result, err = provider.Down(ctx)
|
||||
if errors.Is(err, goose.ErrNoNextVersion) {
|
||||
err = nil
|
||||
break
|
||||
@@ -192,7 +202,7 @@ func StepDown(
|
||||
if err != nil {
|
||||
break
|
||||
}
|
||||
results = append(results, r)
|
||||
results = append(results, result)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -16,6 +16,7 @@ import (
|
||||
// Run with: go test -tags sqlite_fts5 ./...
|
||||
|
||||
func tempDBPath(t *testing.T) string {
|
||||
t.Helper()
|
||||
return filepath.Join(t.TempDir(), "migrate.db")
|
||||
}
|
||||
|
||||
@@ -25,7 +26,9 @@ func newMigrator(t *testing.T) (*goose.Provider, *sql.DB) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { db.Close() })
|
||||
t.Cleanup(func() {
|
||||
db.Close()
|
||||
})
|
||||
m, err := store.NewMigrator(db)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -93,7 +96,8 @@ func TestOpenRejectsNewerSchema(t *testing.T) {
|
||||
}
|
||||
defer db.Close()
|
||||
if _, err := db.Exec(
|
||||
`INSERT INTO goose_db_version (version_id, is_applied) VALUES (9999, 1)`); err != nil {
|
||||
`INSERT INTO goose_db_version (version_id, is_applied) VALUES (9999, 1)`,
|
||||
); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
@@ -268,7 +272,8 @@ func TestNewerDatabaseIsRefused(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := db.Exec(
|
||||
`INSERT INTO goose_db_version (version_id, is_applied) VALUES (9999, 1)`); err != nil {
|
||||
`INSERT INTO goose_db_version (version_id, is_applied) VALUES (9999, 1)`,
|
||||
); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
|
||||
@@ -67,15 +67,20 @@ SELECT t.* FROM opportunities t WHERE t.uuid = ?`
|
||||
// when the mark is set and is negated with NOT to match it unset; an
|
||||
// opportunity with no marks row has every mark unset.
|
||||
const (
|
||||
// listOpportunitiesSelect selects every opportunity, as t.
|
||||
listOpportunitiesSelect = `
|
||||
SELECT t.* FROM opportunities t`
|
||||
|
||||
// listOpportunitiesOrderAndPage orders by latest release, newest first,
|
||||
// and takes a page: the limit and offset arguments.
|
||||
listOpportunitiesOrderAndPage = `
|
||||
ORDER BY t.latest_release_date DESC, t.uuid
|
||||
LIMIT ? OFFSET ?`
|
||||
|
||||
// filterOpportunityStatus matches a status.
|
||||
filterOpportunityStatus = `t.status = ?`
|
||||
|
||||
// filterOpportunityClassificationCode matches a classification code.
|
||||
filterOpportunityClassificationCode = `EXISTS (
|
||||
SELECT 1
|
||||
FROM opportunity_classifications tc
|
||||
@@ -83,35 +88,45 @@ LIMIT ? OFFSET ?`
|
||||
WHERE tc.opportunity_uuid = t.uuid AND c.code = ?
|
||||
)`
|
||||
|
||||
// The argument must come from ftsQuery, never raw user input.
|
||||
// filterOpportunityTitleMatch matches a full-text search of the title. The
|
||||
// argument must come from ftsQuery, never raw user input.
|
||||
filterOpportunityTitleMatch = `t.id IN (
|
||||
SELECT rowid FROM opportunities_fts WHERE opportunities_fts MATCH ?
|
||||
)`
|
||||
|
||||
// filterOpportunityReleasedFrom matches a latest release on or after a
|
||||
// time.
|
||||
filterOpportunityReleasedFrom = `t.latest_release_date >= ?`
|
||||
|
||||
// filterOpportunityReleasedBefore matches a latest release before a time.
|
||||
filterOpportunityReleasedBefore = `t.latest_release_date < ?`
|
||||
|
||||
// filterOpportunityMarkHidden matches the hidden mark set.
|
||||
filterOpportunityMarkHidden = `EXISTS (
|
||||
SELECT 1 FROM opportunity_marks m WHERE m.uuid = t.uuid AND m.hidden
|
||||
)`
|
||||
|
||||
// filterOpportunityMarkSkipped matches the skipped mark set.
|
||||
filterOpportunityMarkSkipped = `EXISTS (
|
||||
SELECT 1 FROM opportunity_marks m WHERE m.uuid = t.uuid AND m.skipped
|
||||
)`
|
||||
|
||||
// filterOpportunityMarkInterested matches the interested mark set.
|
||||
filterOpportunityMarkInterested = `EXISTS (
|
||||
SELECT 1 FROM opportunity_marks m WHERE m.uuid = t.uuid AND m.interested
|
||||
)`
|
||||
|
||||
// filterOpportunityMarkBidSubmitted matches the bid submitted mark set.
|
||||
filterOpportunityMarkBidSubmitted = `EXISTS (
|
||||
SELECT 1 FROM opportunity_marks m WHERE m.uuid = t.uuid AND m.bid_submitted
|
||||
)`
|
||||
|
||||
// filterOpportunityMarkWon matches the won mark set.
|
||||
filterOpportunityMarkWon = `EXISTS (
|
||||
SELECT 1 FROM opportunity_marks m WHERE m.uuid = t.uuid AND m.won
|
||||
)`
|
||||
|
||||
// filterOpportunityMarkLost matches the lost mark set.
|
||||
filterOpportunityMarkLost = `EXISTS (
|
||||
SELECT 1 FROM opportunity_marks m WHERE m.uuid = t.uuid AND m.lost
|
||||
)`
|
||||
|
||||
@@ -84,7 +84,9 @@ func openTestDB(t *testing.T) *sqlx.DB {
|
||||
if err != nil {
|
||||
t.Fatalf("Open: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { s.Close() })
|
||||
t.Cleanup(func() {
|
||||
s.Close()
|
||||
})
|
||||
return s.db
|
||||
}
|
||||
|
||||
|
||||
+188
-132
@@ -103,57 +103,57 @@ func (f OpportunityFilter) limit() int {
|
||||
return f.Limit
|
||||
}
|
||||
|
||||
// prepareOpportunity validates t and normalises it in place before saving: child
|
||||
// rows get the opportunity's UUID and their release id, times are converted to
|
||||
// UTC (so stored timestamps compare correctly as text), and
|
||||
// prepareOpportunity validates opportunity and normalises it in place before
|
||||
// saving: child rows get the opportunity's UUID and their release id, times
|
||||
// are converted to UTC (so stored timestamps compare correctly as text), and
|
||||
// LatestReleaseDate becomes the newest release date in the save.
|
||||
func prepareOpportunity(t *types.Opportunity) error {
|
||||
if t == nil {
|
||||
func prepareOpportunity(opportunity *types.Opportunity) error {
|
||||
if opportunity == nil {
|
||||
return errors.New("store: nil opportunity")
|
||||
}
|
||||
if t.UUID == "" {
|
||||
if opportunity.UUID == "" {
|
||||
return errors.New("store: opportunity has no UUID")
|
||||
}
|
||||
|
||||
t.Marks.UUID = t.UUID
|
||||
if err := t.Marks.Validate(); err != nil {
|
||||
return fmt.Errorf("store: opportunity %s: %w", t.UUID, err)
|
||||
opportunity.Marks.UUID = opportunity.UUID
|
||||
if err := opportunity.Marks.Validate(); err != nil {
|
||||
return fmt.Errorf("store: opportunity %s: %w", opportunity.UUID, err)
|
||||
}
|
||||
|
||||
latest := t.LatestReleaseDate
|
||||
for i := range t.Releases {
|
||||
r := &t.Releases[i]
|
||||
if r.ReleaseID == "" {
|
||||
return fmt.Errorf("store: opportunity %s: release %d has no release id", t.UUID, i)
|
||||
latest := opportunity.LatestReleaseDate
|
||||
for i := range opportunity.Releases {
|
||||
release := &opportunity.Releases[i]
|
||||
if release.ReleaseID == "" {
|
||||
return fmt.Errorf("store: opportunity %s: release %d has no release id", opportunity.UUID, i)
|
||||
}
|
||||
r.UUID = t.UUID
|
||||
r.ReleaseDate = r.ReleaseDate.UTC()
|
||||
if r.ReleaseDate.After(latest) {
|
||||
latest = r.ReleaseDate
|
||||
release.UUID = opportunity.UUID
|
||||
release.ReleaseDate = release.ReleaseDate.UTC()
|
||||
if release.ReleaseDate.After(latest) {
|
||||
latest = release.ReleaseDate
|
||||
}
|
||||
for j := range r.Tags {
|
||||
r.Tags[j].UUID = t.UUID
|
||||
r.Tags[j].ReleaseID = r.ReleaseID
|
||||
for tagIndex := range release.Tags {
|
||||
release.Tags[tagIndex].UUID = opportunity.UUID
|
||||
release.Tags[tagIndex].ReleaseID = release.ReleaseID
|
||||
}
|
||||
}
|
||||
|
||||
if latest.IsZero() {
|
||||
return fmt.Errorf("store: opportunity %s has no release date", t.UUID)
|
||||
return fmt.Errorf("store: opportunity %s has no release date", opportunity.UUID)
|
||||
}
|
||||
|
||||
t.LatestReleaseDate = latest.UTC()
|
||||
t.StartDate = utcPtr(t.StartDate)
|
||||
t.EndDate = utcPtr(t.EndDate)
|
||||
opportunity.LatestReleaseDate = latest.UTC()
|
||||
opportunity.StartDate = utcPtr(opportunity.StartDate)
|
||||
opportunity.EndDate = utcPtr(opportunity.EndDate)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func utcPtr(t *time.Time) *time.Time {
|
||||
if t == nil {
|
||||
func utcPtr(when *time.Time) *time.Time {
|
||||
if when == nil {
|
||||
return nil
|
||||
}
|
||||
u := t.UTC()
|
||||
return &u
|
||||
utc := when.UTC()
|
||||
return &utc
|
||||
}
|
||||
|
||||
// ftsQuery turns free text into an FTS5 query that requires every word, each
|
||||
@@ -162,11 +162,11 @@ func utcPtr(t *time.Time) *time.Time {
|
||||
// FTS5 syntax (operators, column filters, unbalanced quotes). It returns ""
|
||||
// when the text contains no words.
|
||||
func ftsQuery(text string) string {
|
||||
words := strings.FieldsFunc(text, func(r rune) bool {
|
||||
return !unicode.IsLetter(r) && !unicode.IsNumber(r)
|
||||
words := strings.FieldsFunc(text, func(char rune) bool {
|
||||
return !unicode.IsLetter(char) && !unicode.IsNumber(char)
|
||||
})
|
||||
for i, w := range words {
|
||||
words[i] = `"` + w + `"*`
|
||||
for i, word := range words {
|
||||
words[i] = `"` + word + `"*`
|
||||
}
|
||||
return strings.Join(words, " ")
|
||||
}
|
||||
@@ -179,9 +179,9 @@ func ftsQuery(text string) string {
|
||||
//
|
||||
// A database file that doesn't exist is created, and then needs migrating.
|
||||
func Open(ctx context.Context, path string, opts ...Option) (*Store, error) {
|
||||
var o options
|
||||
for _, opt := range opts {
|
||||
opt(&o)
|
||||
var settings options
|
||||
for _, option := range opts {
|
||||
option(&settings)
|
||||
}
|
||||
|
||||
sqldb, err := OpenDB(path)
|
||||
@@ -191,10 +191,12 @@ func Open(ctx context.Context, path string, opts ...Option) (*Store, error) {
|
||||
if err := sqldb.PingContext(ctx); err != nil {
|
||||
return nil, errors.Join(err, sqldb.Close())
|
||||
}
|
||||
if err := prepareSchema(ctx, sqldb, o); err != nil {
|
||||
if err := prepareSchema(ctx, sqldb, settings); err != nil {
|
||||
return nil, errors.Join(err, sqldb.Close())
|
||||
}
|
||||
return &Store{db: sqlx.NewDb(sqldb, "sqlite3")}, nil
|
||||
return &Store{
|
||||
db: sqlx.NewDb(sqldb, "sqlite3"),
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Close closes the database.
|
||||
@@ -214,20 +216,24 @@ type opportunityRow struct {
|
||||
types.Opportunity
|
||||
}
|
||||
|
||||
// SaveOpportunity stores t and its releases in one transaction.
|
||||
// SaveOpportunity stores opportunity and its releases in one transaction.
|
||||
//
|
||||
// Releases (and their tags) are write-once: ones already stored are skipped.
|
||||
// The opportunity row and its classifications are replaced only if t's newest
|
||||
// release is at least as new as the release the stored opportunity came from, so
|
||||
// a late-arriving older release never undoes a newer one.
|
||||
// The opportunity row and its classifications are replaced only if the
|
||||
// opportunity's newest release is at least as new as the release the stored
|
||||
// opportunity came from, so a late-arriving older release never undoes a
|
||||
// newer one.
|
||||
//
|
||||
// t.Marks are stored only for a new opportunity; existing marks are kept (use
|
||||
// SetMarks to change them).
|
||||
// opportunity.Marks are stored only for a new opportunity; existing marks are
|
||||
// kept (use SetMarks to change them).
|
||||
//
|
||||
// t is normalised in place (see prepareOpportunity), and when the opportunity is
|
||||
// opportunity is normalised in place (see prepareOpportunity), and when it is
|
||||
// written its classifications' IDs are filled in.
|
||||
func (s *Store) SaveOpportunity(ctx context.Context, t *types.Opportunity) (SaveResult, error) {
|
||||
if err := prepareOpportunity(t); err != nil {
|
||||
func (s *Store) SaveOpportunity(
|
||||
ctx context.Context,
|
||||
opportunity *types.Opportunity,
|
||||
) (SaveResult, error) {
|
||||
if err := prepareOpportunity(opportunity); err != nil {
|
||||
return SaveResult{}, err
|
||||
}
|
||||
|
||||
@@ -237,7 +243,7 @@ func (s *Store) SaveOpportunity(ctx context.Context, t *types.Opportunity) (Save
|
||||
}
|
||||
defer tx.Rollback() //nolint:errcheck // no-op after Commit
|
||||
|
||||
res, _, err := saveOpportunity(ctx, tx, t)
|
||||
res, _, err := saveOpportunity(ctx, tx, opportunity)
|
||||
if err != nil {
|
||||
return SaveResult{}, err
|
||||
}
|
||||
@@ -247,23 +253,25 @@ func (s *Store) SaveOpportunity(ctx context.Context, t *types.Opportunity) (Save
|
||||
return res, nil
|
||||
}
|
||||
|
||||
// SaveOpportunities stores t and its releases in one transaction.
|
||||
// SaveOpportunities stores each of opportunities and its releases, all in one
|
||||
// transaction, as SaveOpportunity does for one.
|
||||
//
|
||||
// Releases (and their tags) are write-once: ones already stored are skipped.
|
||||
// The opportunity row and its classifications are replaced only if t's newest
|
||||
// release is at least as new as the release the stored opportunity came from, so
|
||||
// a late-arriving older release never undoes a newer one.
|
||||
// An opportunity row and its classifications are replaced only if the
|
||||
// opportunity's newest release is at least as new as the release the stored
|
||||
// opportunity came from, so a late-arriving older release never undoes a
|
||||
// newer one.
|
||||
//
|
||||
// t.Marks are stored only for a new opportunity; existing marks are kept (use
|
||||
// Marks are stored only for a new opportunity; existing marks are kept (use
|
||||
// SetMarks to change them).
|
||||
//
|
||||
// t is normalised in place (see prepareOpportunity), and when the opportunity is
|
||||
// written its classifications' IDs are filled in.
|
||||
// Each opportunity is normalised in place (see prepareOpportunity), and when
|
||||
// it is written its classifications' IDs are filled in.
|
||||
func (s *Store) SaveOpportunities(
|
||||
ctx context.Context, opportunities []*types.Opportunity,
|
||||
) (SaveMultipleResult, error) {
|
||||
for _, t := range opportunities {
|
||||
if err := prepareOpportunity(t); err != nil {
|
||||
for _, opportunity := range opportunities {
|
||||
if err := prepareOpportunity(opportunity); err != nil {
|
||||
return SaveMultipleResult{}, err
|
||||
}
|
||||
}
|
||||
@@ -274,9 +282,11 @@ func (s *Store) SaveOpportunities(
|
||||
}
|
||||
defer tx.Rollback() //nolint:errcheck // no-op after Commit
|
||||
|
||||
allRes := SaveMultipleResult{OpportunityResults: make([]SaveResult, len(opportunities))}
|
||||
for i, t := range opportunities {
|
||||
res, isNew, err := saveOpportunity(ctx, tx, t)
|
||||
allRes := SaveMultipleResult{
|
||||
OpportunityResults: make([]SaveResult, len(opportunities)),
|
||||
}
|
||||
for i, opportunity := range opportunities {
|
||||
res, isNew, err := saveOpportunity(ctx, tx, opportunity)
|
||||
if err != nil {
|
||||
return SaveMultipleResult{}, err
|
||||
}
|
||||
@@ -300,27 +310,35 @@ func (s *Store) SaveOpportunities(
|
||||
// one prepared opportunity inside tx. isNew reports that the opportunity was
|
||||
// not already stored.
|
||||
func saveOpportunity(
|
||||
ctx context.Context, tx *sqlx.Tx, t *types.Opportunity,
|
||||
ctx context.Context, tx *sqlx.Tx, opportunity *types.Opportunity,
|
||||
) (res SaveResult, isNew bool, err error) {
|
||||
var stored time.Time
|
||||
err = tx.GetContext(ctx, &stored, selectOpportunityLatestReleaseDate, t.UUID)
|
||||
err = tx.GetContext(ctx, &stored, selectOpportunityLatestReleaseDate, opportunity.UUID)
|
||||
switch {
|
||||
case errors.Is(err, sql.ErrNoRows):
|
||||
isNew = true
|
||||
res.OpportunityUpdated = true
|
||||
case err != nil:
|
||||
return SaveResult{}, false, fmt.Errorf("store: reading opportunity %s: %w", t.UUID, err)
|
||||
return SaveResult{}, false, fmt.Errorf(
|
||||
"store: reading opportunity %s: %w",
|
||||
opportunity.UUID,
|
||||
err,
|
||||
)
|
||||
default:
|
||||
res.OpportunityUpdated = !t.LatestReleaseDate.Before(stored)
|
||||
res.OpportunityUpdated = !opportunity.LatestReleaseDate.Before(stored)
|
||||
}
|
||||
if res.OpportunityUpdated {
|
||||
if err := writeOpportunity(ctx, tx, t); err != nil {
|
||||
return SaveResult{}, false, fmt.Errorf("store: writing opportunity %s: %w", t.UUID, err)
|
||||
if err := writeOpportunity(ctx, tx, opportunity); err != nil {
|
||||
return SaveResult{}, false, fmt.Errorf(
|
||||
"store: writing opportunity %s: %w",
|
||||
opportunity.UUID,
|
||||
err,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
for _, r := range t.Releases {
|
||||
inserted, err := insertReleaseWithTags(ctx, tx, r)
|
||||
for _, release := range opportunity.Releases {
|
||||
inserted, err := insertReleaseWithTags(ctx, tx, release)
|
||||
if err != nil {
|
||||
return SaveResult{}, false, err
|
||||
}
|
||||
@@ -331,69 +349,86 @@ func saveOpportunity(
|
||||
return res, isNew, nil
|
||||
}
|
||||
|
||||
// insertReleaseWithTags stores r and its tags unless r is already stored, and
|
||||
// reports whether it was inserted.
|
||||
// insertReleaseWithTags stores release and its tags unless it is already
|
||||
// stored, and reports whether it was inserted.
|
||||
func insertReleaseWithTags(
|
||||
ctx context.Context, tx *sqlx.Tx, r types.OpportunityRelease,
|
||||
ctx context.Context, tx *sqlx.Tx, release types.OpportunityRelease,
|
||||
) (bool, error) {
|
||||
result, err := tx.NamedExecContext(ctx, insertRelease, r)
|
||||
result, err := tx.NamedExecContext(ctx, insertRelease, release)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("store: inserting release %s/%s: %w", r.UUID, r.ReleaseID, err)
|
||||
return false, fmt.Errorf(
|
||||
"store: inserting release %s/%s: %w",
|
||||
release.UUID,
|
||||
release.ReleaseID,
|
||||
err,
|
||||
)
|
||||
}
|
||||
if n, err := result.RowsAffected(); err != nil || n == 0 {
|
||||
if affected, err := result.RowsAffected(); err != nil || affected == 0 {
|
||||
return false, err // n == 0: already stored
|
||||
}
|
||||
for _, tag := range r.Tags {
|
||||
for _, tag := range release.Tags {
|
||||
if _, err := tx.NamedExecContext(ctx, insertReleaseTag, tag); err != nil {
|
||||
return false, fmt.Errorf(
|
||||
"store: inserting tag for %s/%s: %w", r.UUID, r.ReleaseID, err,
|
||||
"store: inserting tag for %s/%s: %w", release.UUID, release.ReleaseID, err,
|
||||
)
|
||||
}
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// writeOpportunity upserts t's row, gives it a marks row if it has none
|
||||
// writeOpportunity upserts opportunity's row, gives it a marks row if it has none
|
||||
// (existing marks are kept), and replaces its classifications, filling in
|
||||
// their IDs.
|
||||
func writeOpportunity(ctx context.Context, tx *sqlx.Tx, t *types.Opportunity) (err error) {
|
||||
if _, err := tx.NamedExecContext(ctx, upsertOpportunity, t); err != nil {
|
||||
func writeOpportunity(
|
||||
ctx context.Context,
|
||||
tx *sqlx.Tx,
|
||||
opportunity *types.Opportunity,
|
||||
) (err error) {
|
||||
if _, err := tx.NamedExecContext(ctx, upsertOpportunity, opportunity); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := tx.NamedExecContext(ctx, insertOpportunityMarks, t.Marks); err != nil {
|
||||
if _, err := tx.NamedExecContext(ctx, insertOpportunityMarks, opportunity.Marks); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := tx.ExecContext(ctx, deleteOpportunityClassifications, t.UUID); err != nil {
|
||||
if _, err := tx.ExecContext(ctx, deleteOpportunityClassifications, opportunity.UUID); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
stmt, err := tx.PrepareNamedContext(ctx, upsertClassificationCodeReturningID)
|
||||
statement, err := tx.PrepareNamedContext(ctx, upsertClassificationCodeReturningID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() { err = errors.Join(err, stmt.Close()) }()
|
||||
for i := range t.Classifications {
|
||||
c := &t.Classifications[i]
|
||||
if err := stmt.GetContext(ctx, &c.ID, c); err != nil {
|
||||
defer func() {
|
||||
err = errors.Join(err, statement.Close())
|
||||
}()
|
||||
for i := range opportunity.Classifications {
|
||||
classification := &opportunity.Classifications[i]
|
||||
if err := statement.GetContext(ctx, &classification.ID, classification); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := tx.ExecContext(ctx, insertOpportunityClassification, t.UUID, c.ID); err != nil {
|
||||
if _, err := tx.ExecContext(
|
||||
ctx,
|
||||
insertOpportunityClassification,
|
||||
opportunity.UUID,
|
||||
classification.ID,
|
||||
); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// SetMarks replaces the marks of the opportunity m.UUID, or returns ErrNotFound.
|
||||
// Marks that fail m.Validate (e.g. types.ErrWonAndLost) are not stored.
|
||||
func (s *Store) SetMarks(ctx context.Context, m types.OpportunityMarks) error {
|
||||
if m.UUID == "" {
|
||||
// SetMarks replaces the marks of the opportunity marks.UUID, or returns
|
||||
// ErrNotFound. Marks that fail marks.Validate (e.g. types.ErrWonAndLost) are
|
||||
// not stored.
|
||||
func (s *Store) SetMarks(ctx context.Context, marks types.OpportunityMarks) error {
|
||||
if marks.UUID == "" {
|
||||
return errors.New("store: marks have no UUID")
|
||||
}
|
||||
if err := m.Validate(); err != nil {
|
||||
return fmt.Errorf("store: opportunity %s: %w", m.UUID, err)
|
||||
if err := marks.Validate(); err != nil {
|
||||
return fmt.Errorf("store: opportunity %s: %w", marks.UUID, err)
|
||||
}
|
||||
_, err := s.db.NamedExecContext(ctx, upsertOpportunityMarks, m)
|
||||
_, err := s.db.NamedExecContext(ctx, upsertOpportunityMarks, marks)
|
||||
if isForeignKeyError(err) {
|
||||
return ErrNotFound
|
||||
}
|
||||
@@ -403,8 +438,8 @@ func (s *Store) SetMarks(ctx context.Context, m types.OpportunityMarks) error {
|
||||
// isForeignKeyError reports whether err is SQLite rejecting a write that
|
||||
// breaks a foreign key, e.g. one referring to a missing opportunity.
|
||||
func isForeignKeyError(err error) bool {
|
||||
var se sqlite3.Error
|
||||
return errors.As(err, &se) && se.ExtendedCode == sqlite3.ErrConstraintForeignKey
|
||||
var sqliteErr sqlite3.Error
|
||||
return errors.As(err, &sqliteErr) && sqliteErr.ExtendedCode == sqlite3.ErrConstraintForeignKey
|
||||
}
|
||||
|
||||
// GetOpportunity returns the opportunity with its classifications, marks and
|
||||
@@ -431,13 +466,14 @@ func (s *Store) GetOpportunity(ctx context.Context, uuid string) (*types.Opportu
|
||||
return &opportunities[0], nil
|
||||
}
|
||||
|
||||
// ListOpportunities returns opportunities matching f, with their classifications
|
||||
// and marks but without releases (use ListReleases or GetOpportunity for those).
|
||||
// ListOpportunities returns opportunities matching filter, with their
|
||||
// classifications and marks but without releases (use ListReleases or
|
||||
// GetOpportunity for those).
|
||||
func (s *Store) ListOpportunities(
|
||||
ctx context.Context,
|
||||
f OpportunityFilter,
|
||||
filter OpportunityFilter,
|
||||
) ([]types.Opportunity, error) {
|
||||
query, args := listOpportunitiesQuery(f)
|
||||
query, args := listOpportunitiesQuery(filter)
|
||||
var rows []opportunityRow
|
||||
if err := s.db.SelectContext(ctx, &rows, query, args...); err != nil {
|
||||
return nil, err
|
||||
@@ -457,44 +493,62 @@ func (s *Store) ListOpportunities(
|
||||
|
||||
// listOpportunitiesQuery assembles the ListOpportunities query from the fragments in
|
||||
// queries.go, with one argument per filter that is set (none for mark filters).
|
||||
func listOpportunitiesQuery(f OpportunityFilter) (string, []any) {
|
||||
func listOpportunitiesQuery(filter OpportunityFilter) (string, []any) {
|
||||
var where []string
|
||||
var args []any
|
||||
add := func(clause string, arg any) {
|
||||
add := func(clause string, argument any) {
|
||||
where = append(where, clause)
|
||||
args = append(args, arg)
|
||||
args = append(args, argument)
|
||||
}
|
||||
if f.Status != "" {
|
||||
add(filterOpportunityStatus, f.Status)
|
||||
if filter.Status != "" {
|
||||
add(filterOpportunityStatus, filter.Status)
|
||||
}
|
||||
if f.ClassificationCode != "" {
|
||||
add(filterOpportunityClassificationCode, f.ClassificationCode)
|
||||
if filter.ClassificationCode != "" {
|
||||
add(filterOpportunityClassificationCode, filter.ClassificationCode)
|
||||
}
|
||||
if q := ftsQuery(f.Title); q != "" {
|
||||
add(filterOpportunityTitleMatch, q)
|
||||
if match := ftsQuery(filter.Title); match != "" {
|
||||
add(filterOpportunityTitleMatch, match)
|
||||
}
|
||||
if !f.ReleasedFrom.IsZero() {
|
||||
add(filterOpportunityReleasedFrom, f.ReleasedFrom.UTC())
|
||||
if !filter.ReleasedFrom.IsZero() {
|
||||
add(filterOpportunityReleasedFrom, filter.ReleasedFrom.UTC())
|
||||
}
|
||||
if !f.ReleasedBefore.IsZero() {
|
||||
add(filterOpportunityReleasedBefore, f.ReleasedBefore.UTC())
|
||||
if !filter.ReleasedBefore.IsZero() {
|
||||
add(filterOpportunityReleasedBefore, filter.ReleasedBefore.UTC())
|
||||
}
|
||||
for _, m := range []struct {
|
||||
for _, mark := range []struct {
|
||||
filter MarkFilter
|
||||
clause string
|
||||
}{
|
||||
{f.Hidden, filterOpportunityMarkHidden},
|
||||
{f.Skipped, filterOpportunityMarkSkipped},
|
||||
{f.Interested, filterOpportunityMarkInterested},
|
||||
{f.BidSubmitted, filterOpportunityMarkBidSubmitted},
|
||||
{f.Won, filterOpportunityMarkWon},
|
||||
{f.Lost, filterOpportunityMarkLost},
|
||||
{
|
||||
filter: filter.Hidden,
|
||||
clause: filterOpportunityMarkHidden,
|
||||
},
|
||||
{
|
||||
filter: filter.Skipped,
|
||||
clause: filterOpportunityMarkSkipped,
|
||||
},
|
||||
{
|
||||
filter: filter.Interested,
|
||||
clause: filterOpportunityMarkInterested,
|
||||
},
|
||||
{
|
||||
filter: filter.BidSubmitted,
|
||||
clause: filterOpportunityMarkBidSubmitted,
|
||||
},
|
||||
{
|
||||
filter: filter.Won,
|
||||
clause: filterOpportunityMarkWon,
|
||||
},
|
||||
{
|
||||
filter: filter.Lost,
|
||||
clause: filterOpportunityMarkLost,
|
||||
},
|
||||
} {
|
||||
switch m.filter {
|
||||
switch mark.filter {
|
||||
case MarkSet:
|
||||
where = append(where, m.clause)
|
||||
where = append(where, mark.clause)
|
||||
case MarkUnset:
|
||||
where = append(where, "NOT "+m.clause)
|
||||
where = append(where, "NOT "+mark.clause)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -503,7 +557,7 @@ func listOpportunitiesQuery(f OpportunityFilter) (string, []any) {
|
||||
query += "\nWHERE " + strings.Join(where, "\n\tAND ")
|
||||
}
|
||||
query += listOpportunitiesOrderAndPage
|
||||
return query, append(args, f.limit(), f.Offset)
|
||||
return query, append(args, filter.limit(), filter.Offset)
|
||||
}
|
||||
|
||||
// loadClassifications fills in Classifications for all opportunities with one query.
|
||||
@@ -530,9 +584,9 @@ func (s *Store) loadClassifications(ctx context.Context, opportunities []types.O
|
||||
if err := s.db.SelectContext(ctx, &rows, s.db.Rebind(query), args...); err != nil {
|
||||
return err
|
||||
}
|
||||
for _, r := range rows {
|
||||
t := byUUID[r.OpportunityUUID]
|
||||
t.Classifications = append(t.Classifications, r.ClassificationCode)
|
||||
for _, releaseRow := range rows {
|
||||
opportunity := byUUID[releaseRow.OpportunityUUID]
|
||||
opportunity.Classifications = append(opportunity.Classifications, releaseRow.ClassificationCode)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -548,7 +602,9 @@ func (s *Store) loadMarks(ctx context.Context, opportunities []types.Opportunity
|
||||
for i := range opportunities {
|
||||
uuids[i] = opportunities[i].UUID
|
||||
byUUID[opportunities[i].UUID] = &opportunities[i]
|
||||
opportunities[i].Marks = types.OpportunityMarks{UUID: opportunities[i].UUID}
|
||||
opportunities[i].Marks = types.OpportunityMarks{
|
||||
UUID: opportunities[i].UUID,
|
||||
}
|
||||
}
|
||||
|
||||
query, args, err := sqlx.In(selectMarksForOpportunities, uuids)
|
||||
@@ -559,8 +615,8 @@ func (s *Store) loadMarks(ctx context.Context, opportunities []types.Opportunity
|
||||
if err := s.db.SelectContext(ctx, &rows, s.db.Rebind(query), args...); err != nil {
|
||||
return err
|
||||
}
|
||||
for _, m := range rows {
|
||||
byUUID[m.UUID].Marks = m
|
||||
for _, row := range rows {
|
||||
byUUID[row.UUID].Marks = row
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -581,8 +637,8 @@ func (s *Store) ListReleases(ctx context.Context, uuid string) ([]types.Opportun
|
||||
return nil, err
|
||||
}
|
||||
for _, tag := range tags {
|
||||
if r := byID[tag.ReleaseID]; r != nil {
|
||||
r.Tags = append(r.Tags, tag)
|
||||
if release := byID[tag.ReleaseID]; release != nil {
|
||||
release.Tags = append(release.Tags, tag)
|
||||
}
|
||||
}
|
||||
return releases, nil
|
||||
|
||||
+155
-46
@@ -20,7 +20,9 @@ func openStore(t *testing.T) *store.Store {
|
||||
if err != nil {
|
||||
t.Fatalf("Open: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { s.Close() })
|
||||
t.Cleanup(func() {
|
||||
s.Close()
|
||||
})
|
||||
return s
|
||||
}
|
||||
|
||||
@@ -51,7 +53,10 @@ func opportunity(uuid, title string, releaseDay int, releaseIDs ...string) *type
|
||||
{IsCPV: true, Code: "03000000", Classification: "Agricultural products"},
|
||||
},
|
||||
// No UUID: SaveOpportunity must fill it in.
|
||||
Marks: types.OpportunityMarks{Interested: true, BidSubmitted: true},
|
||||
Marks: types.OpportunityMarks{
|
||||
Interested: true,
|
||||
BidSubmitted: true,
|
||||
},
|
||||
}
|
||||
for i, id := range releaseIDs {
|
||||
t.Releases = append(t.Releases, types.OpportunityRelease{
|
||||
@@ -131,7 +136,11 @@ func TestSaveAndGetRoundTrip(t *testing.T) {
|
||||
t.Errorf("classifications: got %+v (leading zero must survive)", got.Classifications)
|
||||
}
|
||||
|
||||
wantMarks := types.OpportunityMarks{UUID: "ocds-1", Interested: true, BidSubmitted: true}
|
||||
wantMarks := types.OpportunityMarks{
|
||||
UUID: "ocds-1",
|
||||
Interested: true,
|
||||
BidSubmitted: true,
|
||||
}
|
||||
if got.Marks != wantMarks {
|
||||
t.Errorf("marks: got %+v, want %+v", got.Marks, wantMarks)
|
||||
}
|
||||
@@ -230,7 +239,9 @@ func TestNewerReleaseUpdatesOpportunityAndSearchIndex(t *testing.T) {
|
||||
}
|
||||
|
||||
for title, want := range map[string]int{"cloud": 0, "redesign": 1} {
|
||||
ts, err := s.ListOpportunities(ctx, store.OpportunityFilter{Title: title})
|
||||
ts, err := s.ListOpportunities(ctx, store.OpportunityFilter{
|
||||
Title: title,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -247,8 +258,11 @@ func TestClassificationRowsAreShared(t *testing.T) {
|
||||
mustSave(t, s, a)
|
||||
mustSave(t, s, b)
|
||||
if a.Classifications[0].ID == 0 || a.Classifications[0].ID != b.Classifications[0].ID {
|
||||
t.Errorf("identical classifications should share an ID: %d vs %d",
|
||||
a.Classifications[0].ID, b.Classifications[0].ID)
|
||||
t.Errorf(
|
||||
"identical classifications should share an ID: %d vs %d",
|
||||
a.Classifications[0].ID,
|
||||
b.Classifications[0].ID,
|
||||
)
|
||||
}
|
||||
|
||||
c := opportunity("ocds-c", "C", 1, "r1")
|
||||
@@ -299,31 +313,59 @@ func TestListOpportunitiesFilters(t *testing.T) {
|
||||
want []string
|
||||
}{
|
||||
{"all, newest first", store.OpportunityFilter{}, []string{"ocds-3", "ocds-2", "ocds-1"}},
|
||||
{"status", store.OpportunityFilter{Status: "complete"}, []string{"ocds-2"}},
|
||||
{"status", store.OpportunityFilter{
|
||||
Status: "complete",
|
||||
}, []string{"ocds-2"}},
|
||||
{
|
||||
"classification",
|
||||
store.OpportunityFilter{ClassificationCode: "72000000"},
|
||||
store.OpportunityFilter{
|
||||
ClassificationCode: "72000000",
|
||||
},
|
||||
[]string{"ocds-3", "ocds-1"},
|
||||
},
|
||||
{"title word", store.OpportunityFilter{Title: "cloud"}, []string{"ocds-3", "ocds-1"}},
|
||||
{"title prefixes", store.OpportunityFilter{Title: "clo migr"}, []string{"ocds-1"}},
|
||||
{"title stemming", store.OpportunityFilter{Title: "migrating"}, []string{"ocds-1"}},
|
||||
{"title case", store.OpportunityFilter{Title: "ROAD"}, []string{"ocds-2"}},
|
||||
{"title word", store.OpportunityFilter{
|
||||
Title: "cloud",
|
||||
}, []string{"ocds-3", "ocds-1"}},
|
||||
{"title prefixes", store.OpportunityFilter{
|
||||
Title: "clo migr",
|
||||
}, []string{"ocds-1"}},
|
||||
{"title stemming", store.OpportunityFilter{
|
||||
Title: "migrating",
|
||||
}, []string{"ocds-1"}},
|
||||
{"title case", store.OpportunityFilter{
|
||||
Title: "ROAD",
|
||||
}, []string{"ocds-2"}},
|
||||
{
|
||||
"title syntax is not FTS syntax",
|
||||
store.OpportunityFilter{Title: `cloud" OR title:road*`},
|
||||
store.OpportunityFilter{
|
||||
Title: `cloud" OR title:road*`,
|
||||
},
|
||||
[]string{},
|
||||
},
|
||||
{
|
||||
"title punctuation only",
|
||||
store.OpportunityFilter{Title: `"*:()`},
|
||||
store.OpportunityFilter{
|
||||
Title: `"*:()`,
|
||||
},
|
||||
[]string{"ocds-3", "ocds-2", "ocds-1"},
|
||||
},
|
||||
{"released from", store.OpportunityFilter{ReleasedFrom: day(2)}, []string{"ocds-3", "ocds-2"}},
|
||||
{"released before", store.OpportunityFilter{ReleasedBefore: day(2)}, []string{"ocds-1"}},
|
||||
{"combined", store.OpportunityFilter{Title: "cloud", ReleasedFrom: day(2)}, []string{"ocds-3"}},
|
||||
{"limit", store.OpportunityFilter{Limit: 2}, []string{"ocds-3", "ocds-2"}},
|
||||
{"offset", store.OpportunityFilter{Limit: 2, Offset: 2}, []string{"ocds-1"}},
|
||||
{"released from", store.OpportunityFilter{
|
||||
ReleasedFrom: day(2),
|
||||
}, []string{"ocds-3", "ocds-2"}},
|
||||
{"released before", store.OpportunityFilter{
|
||||
ReleasedBefore: day(2),
|
||||
}, []string{"ocds-1"}},
|
||||
{"combined", store.OpportunityFilter{
|
||||
Title: "cloud",
|
||||
ReleasedFrom: day(2),
|
||||
}, []string{"ocds-3"}},
|
||||
{"limit", store.OpportunityFilter{
|
||||
Limit: 2,
|
||||
}, []string{"ocds-3", "ocds-2"}},
|
||||
{"offset", store.OpportunityFilter{
|
||||
Limit: 2,
|
||||
Offset: 2,
|
||||
}, []string{"ocds-1"}},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
@@ -337,7 +379,9 @@ func TestListOpportunitiesFilters(t *testing.T) {
|
||||
})
|
||||
}
|
||||
|
||||
got, err := s.ListOpportunities(ctx, store.OpportunityFilter{Status: "complete"})
|
||||
got, err := s.ListOpportunities(ctx, store.OpportunityFilter{
|
||||
Status: "complete",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -413,7 +457,9 @@ func TestReopenKeepsDataAndSkipsAppliedMigrations(t *testing.T) {
|
||||
t.Fatalf("reopen: %v", err)
|
||||
}
|
||||
defer s.Close()
|
||||
got, err := s.ListOpportunities(ctx, store.OpportunityFilter{Title: "cloud"})
|
||||
got, err := s.ListOpportunities(ctx, store.OpportunityFilter{
|
||||
Title: "cloud",
|
||||
})
|
||||
if err != nil || len(got) != 1 {
|
||||
t.Errorf("after reopen: %v, %v", uuids(got), err)
|
||||
}
|
||||
@@ -440,8 +486,12 @@ func TestResaveLoadedOpportunity(t *testing.T) {
|
||||
t.Errorf("resave not applied: %+v", got)
|
||||
}
|
||||
if got.Classifications[0].ID != ids[0] || got.Classifications[1].ID != ids[1] {
|
||||
t.Errorf("classification IDs changed on resave: %v -> %d, %d",
|
||||
ids, got.Classifications[0].ID, got.Classifications[1].ID)
|
||||
t.Errorf(
|
||||
"classification IDs changed on resave: %v -> %d, %d",
|
||||
ids,
|
||||
got.Classifications[0].ID,
|
||||
got.Classifications[1].ID,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -473,14 +523,19 @@ func TestSetMarksErrors(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
if err := s.SetMarks(
|
||||
ctx,
|
||||
types.OpportunityMarks{UUID: "missing", Hidden: true},
|
||||
types.OpportunityMarks{
|
||||
UUID: "missing",
|
||||
Hidden: true,
|
||||
},
|
||||
); !errors.Is(
|
||||
err,
|
||||
store.ErrNotFound,
|
||||
) {
|
||||
t.Errorf("unknown opportunity: err = %v, want ErrNotFound", err)
|
||||
}
|
||||
if err := s.SetMarks(ctx, types.OpportunityMarks{Hidden: true}); err == nil {
|
||||
if err := s.SetMarks(ctx, types.OpportunityMarks{
|
||||
Hidden: true,
|
||||
}); err == nil {
|
||||
t.Errorf("no UUID: expected an error")
|
||||
}
|
||||
}
|
||||
@@ -492,7 +547,11 @@ func TestSaveKeepsExistingMarks(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
mustSave(t, s, opportunity("ocds-1", "T", 1, "r1"))
|
||||
mustSave(t, s, opportunity("ocds-2", "T", 1, "r1"))
|
||||
want := types.OpportunityMarks{UUID: "ocds-1", Hidden: true, Lost: true}
|
||||
want := types.OpportunityMarks{
|
||||
UUID: "ocds-1",
|
||||
Hidden: true,
|
||||
Lost: true,
|
||||
}
|
||||
mustSetMarks(t, s, want)
|
||||
|
||||
newer := opportunity("ocds-1", "Amended", 2, "r2")
|
||||
@@ -518,7 +577,9 @@ func TestSaveOpportunitiesStoresMarks(t *testing.T) {
|
||||
s := openStore(t)
|
||||
a := opportunity("ocds-a", "A", 1, "r1")
|
||||
b := opportunity("ocds-b", "B", 1, "r1")
|
||||
b.Marks = types.OpportunityMarks{Skipped: true}
|
||||
b.Marks = types.OpportunityMarks{
|
||||
Skipped: true,
|
||||
}
|
||||
if _, err := s.SaveOpportunities(context.Background(), []*types.Opportunity{a, b}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -539,8 +600,15 @@ func TestListOpportunitiesMarkFilters(t *testing.T) {
|
||||
mustSave(t, s, opportunity(uuid, "Cloud hosting", i+1, "r1"))
|
||||
}
|
||||
// ocds-1 keeps the marks it was saved with: Interested, BidSubmitted.
|
||||
mustSetMarks(t, s, types.OpportunityMarks{UUID: "ocds-2", Hidden: true})
|
||||
mustSetMarks(t, s, types.OpportunityMarks{UUID: "ocds-3", Skipped: true, Lost: true})
|
||||
mustSetMarks(t, s, types.OpportunityMarks{
|
||||
UUID: "ocds-2",
|
||||
Hidden: true,
|
||||
})
|
||||
mustSetMarks(t, s, types.OpportunityMarks{
|
||||
UUID: "ocds-3",
|
||||
Skipped: true,
|
||||
Lost: true,
|
||||
})
|
||||
mustSetMarks(t, s, types.OpportunityMarks{
|
||||
UUID: "ocds-4", Interested: true, BidSubmitted: true, Won: true,
|
||||
})
|
||||
@@ -552,31 +620,65 @@ func TestListOpportunitiesMarkFilters(t *testing.T) {
|
||||
want []string
|
||||
}{
|
||||
{"any", store.OpportunityFilter{}, []string{"ocds-4", "ocds-3", "ocds-2", "ocds-1"}},
|
||||
{"hidden", store.OpportunityFilter{Hidden: set}, []string{"ocds-2"}},
|
||||
{"not hidden", store.OpportunityFilter{Hidden: unset}, []string{"ocds-4", "ocds-3", "ocds-1"}},
|
||||
{"skipped", store.OpportunityFilter{Skipped: set}, []string{"ocds-3"}},
|
||||
{"hidden", store.OpportunityFilter{
|
||||
Hidden: set,
|
||||
}, []string{"ocds-2"}},
|
||||
{"not hidden", store.OpportunityFilter{
|
||||
Hidden: unset,
|
||||
}, []string{"ocds-4", "ocds-3", "ocds-1"}},
|
||||
{"skipped", store.OpportunityFilter{
|
||||
Skipped: set,
|
||||
}, []string{"ocds-3"}},
|
||||
{
|
||||
"not skipped",
|
||||
store.OpportunityFilter{Skipped: unset},
|
||||
store.OpportunityFilter{
|
||||
Skipped: unset,
|
||||
},
|
||||
[]string{"ocds-4", "ocds-2", "ocds-1"},
|
||||
},
|
||||
{"interested", store.OpportunityFilter{Interested: set}, []string{"ocds-4", "ocds-1"}},
|
||||
{"not interested", store.OpportunityFilter{Interested: unset}, []string{"ocds-3", "ocds-2"}},
|
||||
{"bid submitted", store.OpportunityFilter{BidSubmitted: set}, []string{"ocds-4", "ocds-1"}},
|
||||
{"interested", store.OpportunityFilter{
|
||||
Interested: set,
|
||||
}, []string{"ocds-4", "ocds-1"}},
|
||||
{"not interested", store.OpportunityFilter{
|
||||
Interested: unset,
|
||||
}, []string{"ocds-3", "ocds-2"}},
|
||||
{"bid submitted", store.OpportunityFilter{
|
||||
BidSubmitted: set,
|
||||
}, []string{"ocds-4", "ocds-1"}},
|
||||
{
|
||||
"bid not submitted",
|
||||
store.OpportunityFilter{BidSubmitted: unset},
|
||||
store.OpportunityFilter{
|
||||
BidSubmitted: unset,
|
||||
},
|
||||
[]string{"ocds-3", "ocds-2"},
|
||||
},
|
||||
{"won", store.OpportunityFilter{Won: set}, []string{"ocds-4"}},
|
||||
{"not won", store.OpportunityFilter{Won: unset}, []string{"ocds-3", "ocds-2", "ocds-1"}},
|
||||
{"lost", store.OpportunityFilter{Lost: set}, []string{"ocds-3"}},
|
||||
{"not lost", store.OpportunityFilter{Lost: unset}, []string{"ocds-4", "ocds-2", "ocds-1"}},
|
||||
{"two marks", store.OpportunityFilter{BidSubmitted: set, Won: unset}, []string{"ocds-1"}},
|
||||
{"no match", store.OpportunityFilter{Won: set, Lost: set}, []string{}},
|
||||
{"won", store.OpportunityFilter{
|
||||
Won: set,
|
||||
}, []string{"ocds-4"}},
|
||||
{"not won", store.OpportunityFilter{
|
||||
Won: unset,
|
||||
}, []string{"ocds-3", "ocds-2", "ocds-1"}},
|
||||
{"lost", store.OpportunityFilter{
|
||||
Lost: set,
|
||||
}, []string{"ocds-3"}},
|
||||
{"not lost", store.OpportunityFilter{
|
||||
Lost: unset,
|
||||
}, []string{"ocds-4", "ocds-2", "ocds-1"}},
|
||||
{"two marks", store.OpportunityFilter{
|
||||
BidSubmitted: set,
|
||||
Won: unset,
|
||||
}, []string{"ocds-1"}},
|
||||
{"no match", store.OpportunityFilter{
|
||||
Won: set,
|
||||
Lost: set,
|
||||
}, []string{}},
|
||||
{
|
||||
"with other filters",
|
||||
store.OpportunityFilter{Hidden: unset, Title: "cloud", ReleasedFrom: day(3)},
|
||||
store.OpportunityFilter{
|
||||
Hidden: unset,
|
||||
Title: "cloud",
|
||||
ReleasedFrom: day(3),
|
||||
},
|
||||
[]string{"ocds-4", "ocds-3"},
|
||||
},
|
||||
}
|
||||
@@ -608,7 +710,11 @@ func TestWonAndLostAreExclusive(t *testing.T) {
|
||||
mustSave(t, s, opportunity("ocds-1", "T", 1, "r1"))
|
||||
before := mustGet(t, s, "ocds-1").Marks
|
||||
|
||||
both := types.OpportunityMarks{UUID: "ocds-1", Won: true, Lost: true}
|
||||
both := types.OpportunityMarks{
|
||||
UUID: "ocds-1",
|
||||
Won: true,
|
||||
Lost: true,
|
||||
}
|
||||
if err := s.SetMarks(ctx, both); !errors.Is(err, types.ErrWonAndLost) {
|
||||
t.Errorf("SetMarks(won and lost): err = %v, want ErrWonAndLost", err)
|
||||
}
|
||||
@@ -617,7 +723,10 @@ func TestWonAndLostAreExclusive(t *testing.T) {
|
||||
}
|
||||
|
||||
in := opportunity("ocds-2", "T", 1, "r1")
|
||||
in.Marks = types.OpportunityMarks{Won: true, Lost: true}
|
||||
in.Marks = types.OpportunityMarks{
|
||||
Won: true,
|
||||
Lost: true,
|
||||
}
|
||||
if _, err := s.SaveOpportunity(ctx, in); !errors.Is(err, types.ErrWonAndLost) {
|
||||
t.Errorf("SaveOpportunity(won and lost): err = %v, want ErrWonAndLost", err)
|
||||
}
|
||||
|
||||
+58
-16
@@ -24,33 +24,75 @@ var byteUnits = []struct {
|
||||
name string
|
||||
size int64
|
||||
}{
|
||||
{"EiB", humanize.EiByte}, {"EB", humanize.EByte},
|
||||
{"PiB", humanize.PiByte}, {"PB", humanize.PByte},
|
||||
{"TiB", humanize.TiByte}, {"TB", humanize.TByte},
|
||||
{"GiB", humanize.GiByte}, {"GB", humanize.GByte},
|
||||
{"MiB", humanize.MiByte}, {"MB", humanize.MByte},
|
||||
{"KiB", humanize.KiByte}, {"KB", humanize.KByte},
|
||||
{
|
||||
name: "EiB",
|
||||
size: humanize.EiByte,
|
||||
},
|
||||
{
|
||||
name: "EB",
|
||||
size: humanize.EByte,
|
||||
},
|
||||
{
|
||||
name: "PiB",
|
||||
size: humanize.PiByte,
|
||||
},
|
||||
{
|
||||
name: "PB",
|
||||
size: humanize.PByte,
|
||||
},
|
||||
{
|
||||
name: "TiB",
|
||||
size: humanize.TiByte,
|
||||
},
|
||||
{
|
||||
name: "TB",
|
||||
size: humanize.TByte,
|
||||
},
|
||||
{
|
||||
name: "GiB",
|
||||
size: humanize.GiByte,
|
||||
},
|
||||
{
|
||||
name: "GB",
|
||||
size: humanize.GByte,
|
||||
},
|
||||
{
|
||||
name: "MiB",
|
||||
size: humanize.MiByte,
|
||||
},
|
||||
{
|
||||
name: "MB",
|
||||
size: humanize.MByte,
|
||||
},
|
||||
{
|
||||
name: "KiB",
|
||||
size: humanize.KiByte,
|
||||
},
|
||||
{
|
||||
name: "KB",
|
||||
size: humanize.KByte,
|
||||
},
|
||||
}
|
||||
|
||||
// ParseByteSize parses a size such as "32KiB", "1 MB" or "4096". Unit names
|
||||
// are case-insensitive. "-1" (or "-1B") is ByteSizeNoLimit; any other
|
||||
// negative size is an error.
|
||||
func ParseByteSize(s string) (ByteSize, error) {
|
||||
trimmed := strings.TrimSpace(s)
|
||||
func ParseByteSize(text string) (ByteSize, error) {
|
||||
trimmed := strings.TrimSpace(text)
|
||||
if trimmed == "-1" || strings.EqualFold(trimmed, "-1B") {
|
||||
return ByteSizeNoLimit, nil
|
||||
}
|
||||
if strings.HasPrefix(trimmed, "-") {
|
||||
return 0, fmt.Errorf("byte size %q is negative; only -1 (no limit) is allowed", s)
|
||||
return 0, fmt.Errorf("byte size %q is negative; only -1 (no limit) is allowed", text)
|
||||
}
|
||||
n, err := humanize.ParseBytes(s)
|
||||
parsed, err := humanize.ParseBytes(text)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if n > math.MaxInt64 {
|
||||
return 0, fmt.Errorf("byte size %q is too large", s)
|
||||
if parsed > math.MaxInt64 {
|
||||
return 0, fmt.Errorf("byte size %q is too large", text)
|
||||
}
|
||||
return ByteSize(n), nil
|
||||
return ByteSize(parsed), nil
|
||||
}
|
||||
|
||||
// String formats b exactly, in the largest unit that divides it (e.g. "32KiB",
|
||||
@@ -61,9 +103,9 @@ func (b ByteSize) String() string {
|
||||
return "-1"
|
||||
}
|
||||
if b != 0 {
|
||||
for _, u := range byteUnits {
|
||||
if int64(b)%u.size == 0 {
|
||||
return strconv.FormatInt(int64(b)/u.size, 10) + u.name
|
||||
for _, unit := range byteUnits {
|
||||
if int64(b)%unit.size == 0 {
|
||||
return strconv.FormatInt(int64(b)/unit.size, 10) + unit.name
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+15
-12
@@ -135,26 +135,29 @@ func (w Worker) execute(
|
||||
}
|
||||
}
|
||||
|
||||
// AwaitTask runs fn in its own goroutine and returns its result, or ctx.Err()
|
||||
// AwaitTask runs task in its own goroutine and returns its result, or ctx.Err()
|
||||
// if ctx is cancelled first. This lets a Task stop waiting on a call that does
|
||||
// not respond promptly to cancellation.
|
||||
//
|
||||
// The result channel is buffered, so if ctx wins the race the goroutine can
|
||||
// still deliver its result and exit instead of blocking forever.
|
||||
func AwaitTask[T any](ctx context.Context, fn func() (T, error)) (T, error) {
|
||||
func AwaitTask[T any](ctx context.Context, task func() (T, error)) (T, error) {
|
||||
type result struct {
|
||||
val T
|
||||
err error
|
||||
}
|
||||
ch := make(chan result, 1)
|
||||
results := make(chan result, 1)
|
||||
go func() {
|
||||
val, err := fn()
|
||||
ch <- result{val, err}
|
||||
value, err := task()
|
||||
results <- result{
|
||||
val: value,
|
||||
err: err,
|
||||
}
|
||||
}()
|
||||
|
||||
select {
|
||||
case r := <-ch:
|
||||
return r.val, r.err
|
||||
case result := <-results:
|
||||
return result.val, result.err
|
||||
case <-ctx.Done():
|
||||
var zero T
|
||||
return zero, ctx.Err()
|
||||
@@ -162,15 +165,15 @@ func AwaitTask[T any](ctx context.Context, fn func() (T, error)) (T, error) {
|
||||
}
|
||||
|
||||
// AwaitTaskE is AwaitTask for a function that returns only an error.
|
||||
func AwaitTaskE(ctx context.Context, fn func() error) error {
|
||||
ch := make(chan error, 1)
|
||||
func AwaitTaskE(ctx context.Context, task func() error) error {
|
||||
results := make(chan error, 1)
|
||||
go func() {
|
||||
err := fn()
|
||||
ch <- err
|
||||
err := task()
|
||||
results <- err
|
||||
}()
|
||||
|
||||
select {
|
||||
case err := <-ch:
|
||||
case err := <-results:
|
||||
return err
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
|
||||
@@ -19,14 +19,26 @@ func TestMain(m *testing.M) {
|
||||
}
|
||||
|
||||
func TestWorkerRunRejectsBadConfig(t *testing.T) {
|
||||
noop := func(context.Context, time.Time) error { return nil }
|
||||
noop := func(context.Context, time.Time) error {
|
||||
return nil
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
w Worker
|
||||
}{
|
||||
{"no task", Worker{Name: "a", Interval: time.Second}},
|
||||
{"zero interval", Worker{Name: "b", Task: noop}},
|
||||
{"negative interval", Worker{Name: "c", Interval: -time.Second, Task: noop}},
|
||||
{"no task", Worker{
|
||||
Name: "a",
|
||||
Interval: time.Second,
|
||||
}},
|
||||
{"zero interval", Worker{
|
||||
Name: "b",
|
||||
Task: noop,
|
||||
}},
|
||||
{"negative interval", Worker{
|
||||
Name: "c",
|
||||
Interval: -time.Second,
|
||||
Task: noop,
|
||||
}},
|
||||
}
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
@@ -90,7 +102,9 @@ func TestWorkerRunCheckpoints(t *testing.T) {
|
||||
}
|
||||
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- w.Run(ctx, since0) }()
|
||||
go func() {
|
||||
done <- w.Run(ctx, since0)
|
||||
}()
|
||||
select {
|
||||
case err := <-done:
|
||||
if err != nil {
|
||||
@@ -186,7 +200,11 @@ func TestWorkerExecute(t *testing.T) {
|
||||
}
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
w := Worker{Task: func(context.Context, time.Time) error { return tc.err }}
|
||||
w := Worker{
|
||||
Task: func(context.Context, time.Time) error {
|
||||
return tc.err
|
||||
},
|
||||
}
|
||||
before := time.Now()
|
||||
got := w.execute(context.Background(), logger, since)
|
||||
if tc.advance {
|
||||
@@ -202,11 +220,15 @@ func TestWorkerExecute(t *testing.T) {
|
||||
|
||||
func TestAwaitTask(t *testing.T) {
|
||||
errFail := errors.New("boom")
|
||||
v, err := AwaitTask(context.Background(), func() (int, error) { return 42, nil })
|
||||
v, err := AwaitTask(context.Background(), func() (int, error) {
|
||||
return 42, nil
|
||||
})
|
||||
if v != 42 || err != nil {
|
||||
t.Fatalf("got (%d, %v), want (42, nil)", v, err)
|
||||
}
|
||||
v, err = AwaitTask(context.Background(), func() (int, error) { return 7, errFail })
|
||||
v, err = AwaitTask(context.Background(), func() (int, error) {
|
||||
return 7, errFail
|
||||
})
|
||||
if v != 7 || !errors.Is(err, errFail) {
|
||||
t.Fatalf("got (%d, %v), want (7, %v)", v, err, errFail)
|
||||
}
|
||||
@@ -214,8 +236,14 @@ func TestAwaitTask(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
release := make(chan struct{})
|
||||
defer close(release)
|
||||
go func() { time.Sleep(5 * time.Millisecond); cancel() }()
|
||||
v, err = AwaitTask(ctx, func() (int, error) { <-release; return 1, nil })
|
||||
go func() {
|
||||
time.Sleep(5 * time.Millisecond)
|
||||
cancel()
|
||||
}()
|
||||
v, err = AwaitTask(ctx, func() (int, error) {
|
||||
<-release
|
||||
return 1, nil
|
||||
})
|
||||
if v != 0 || !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("got (%d, %v), want (0, context.Canceled)", v, err)
|
||||
}
|
||||
@@ -223,12 +251,16 @@ func TestAwaitTask(t *testing.T) {
|
||||
|
||||
func TestAwaitTaskE(t *testing.T) {
|
||||
errFail := errors.New("boom")
|
||||
if err := AwaitTaskE(context.Background(), func() error { return nil }); err != nil {
|
||||
if err := AwaitTaskE(context.Background(), func() error {
|
||||
return nil
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := AwaitTaskE(
|
||||
context.Background(),
|
||||
func() error { return errFail },
|
||||
func() error {
|
||||
return errFail
|
||||
},
|
||||
); !errors.Is(
|
||||
err,
|
||||
errFail,
|
||||
@@ -242,7 +274,10 @@ func TestAwaitTaskE(t *testing.T) {
|
||||
defer close(release)
|
||||
if err := AwaitTaskE(
|
||||
ctx,
|
||||
func() error { <-release; return nil },
|
||||
func() error {
|
||||
<-release
|
||||
return nil
|
||||
},
|
||||
); !errors.Is(
|
||||
err,
|
||||
context.DeadlineExceeded,
|
||||
|
||||
@@ -1,8 +1,11 @@
|
||||
package version
|
||||
|
||||
// Version information (set at build time with -ldflags)
|
||||
// Version information (set at build time with -ldflags).
|
||||
var (
|
||||
Version = "dev"
|
||||
// Version is the release version, or "dev" for a local build.
|
||||
Version = "dev"
|
||||
// BuildTime is when the binary was built.
|
||||
BuildTime = "unknown"
|
||||
// GitCommit is the commit the binary was built from.
|
||||
GitCommit = "unknown"
|
||||
)
|
||||
@@ -55,18 +55,18 @@ var errUnsupportedData = errors.New("binary frames are not supported")
|
||||
// runs with; the client calls it when it finishes or is closed. The client's
|
||||
// messages are limited by websocket.message_rate_limit if it is enabled.
|
||||
func newClient(
|
||||
h *Hub, conn msgConn, ip string, cancel context.CancelFunc,
|
||||
hub *Hub, conn msgConn, ip string, cancel context.CancelFunc,
|
||||
) *client {
|
||||
limiter := rate.NewLimiter(rate.Inf, 0)
|
||||
if rule := h.conf.MessageRateLimit; rule.Enabled && rule.PerSecond > 0 {
|
||||
if rule := hub.conf.MessageRateLimit; rule.Enabled && rule.PerSecond > 0 {
|
||||
limiter = rate.NewLimiter(
|
||||
rate.Limit(rule.PerSecond),
|
||||
max(rule.Burst, 1),
|
||||
)
|
||||
}
|
||||
|
||||
c := &client{
|
||||
hub: h,
|
||||
client := &client{
|
||||
hub: hub,
|
||||
conn: conn,
|
||||
id: rand.Text()[:12],
|
||||
ip: ip,
|
||||
@@ -74,9 +74,9 @@ func newClient(
|
||||
cancel: cancel,
|
||||
limiter: limiter,
|
||||
}
|
||||
c.lastSeen.Store(time.Now().UnixNano())
|
||||
client.lastSeen.Store(time.Now().UnixNano())
|
||||
|
||||
return c
|
||||
return client
|
||||
}
|
||||
|
||||
// run starts the writer and keepalive goroutines and reads until the
|
||||
@@ -84,8 +84,14 @@ func newClient(
|
||||
func (c *client) run(ctx context.Context) {
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(2)
|
||||
go func() { defer wg.Done(); c.writeLoop(ctx) }()
|
||||
go func() { defer wg.Done(); c.pingLoop(ctx) }()
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
c.writeLoop(ctx)
|
||||
}()
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
c.pingLoop(ctx)
|
||||
}()
|
||||
|
||||
c.sendEnvelope(Envelope{
|
||||
Type: TypeWelcome,
|
||||
@@ -134,35 +140,35 @@ func (c *client) readLoop(ctx context.Context) error {
|
||||
continue
|
||||
}
|
||||
// Unpack the envelope and pass to handle to process if valid
|
||||
var env Envelope
|
||||
if err := json.Unmarshal(data, &env); err != nil {
|
||||
var envelope Envelope
|
||||
if err := json.Unmarshal(data, &envelope); err != nil {
|
||||
c.sendError("", "invalid JSON envelope")
|
||||
continue
|
||||
}
|
||||
c.handle(env)
|
||||
c.handle(envelope)
|
||||
}
|
||||
}
|
||||
|
||||
// handle answers one envelope from the client: ping and echo, or an error
|
||||
// for any other type.
|
||||
func (c *client) handle(env Envelope) {
|
||||
switch env.Type {
|
||||
func (c *client) handle(envelope Envelope) {
|
||||
switch envelope.Type {
|
||||
case TypePing:
|
||||
c.sendEnvelope(Envelope{
|
||||
Type: TypePong,
|
||||
ID: env.ID,
|
||||
ID: envelope.ID,
|
||||
Payload: mustRaw(map[string]any{"time": time.Now().UTC()}),
|
||||
})
|
||||
|
||||
case TypeEcho:
|
||||
c.sendEnvelope(Envelope{
|
||||
Type: TypeEcho,
|
||||
ID: env.ID,
|
||||
Payload: env.Payload,
|
||||
ID: envelope.ID,
|
||||
Payload: envelope.Payload,
|
||||
})
|
||||
|
||||
default:
|
||||
c.sendError(env.ID, "unknown message type")
|
||||
c.sendError(envelope.ID, "unknown message type")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -235,10 +241,10 @@ func (c *client) trySend(msg []byte) bool {
|
||||
}
|
||||
}
|
||||
|
||||
// sendEnvelope queues env for the client, closing the client as a slow
|
||||
// sendEnvelope queues envelope for the client, closing the client as a slow
|
||||
// consumer if its queue is full.
|
||||
func (c *client) sendEnvelope(env Envelope) {
|
||||
encoded, err := json.Marshal(env)
|
||||
func (c *client) sendEnvelope(envelope Envelope) {
|
||||
encoded, err := json.Marshal(envelope)
|
||||
if err != nil {
|
||||
c.hub.logger().Error("marshal", "client", c.id, "error", err)
|
||||
return
|
||||
@@ -298,7 +304,8 @@ func withTimeout(
|
||||
ctx context.Context, timeout time.Duration,
|
||||
) (context.Context, context.CancelFunc) {
|
||||
if timeout <= 0 {
|
||||
return ctx, func() {}
|
||||
return ctx, func() {
|
||||
}
|
||||
}
|
||||
return context.WithTimeout(ctx, timeout)
|
||||
}
|
||||
@@ -23,7 +23,11 @@ type fakeConn struct {
|
||||
closed chan ws.StatusCode
|
||||
}
|
||||
|
||||
func newFakeConn() *fakeConn { return &fakeConn{closed: make(chan ws.StatusCode, 1)} }
|
||||
func newFakeConn() *fakeConn {
|
||||
return &fakeConn{
|
||||
closed: make(chan ws.StatusCode, 1),
|
||||
}
|
||||
}
|
||||
|
||||
func (f *fakeConn) Read(ctx context.Context) (ws.MessageType, []byte, error) {
|
||||
<-ctx.Done()
|
||||
@@ -52,7 +56,10 @@ func newTestClient(f *fakeConn) (*client, context.Context) {
|
||||
// An envelope that can't be marshalled is dropped, not queued.
|
||||
func TestSendEnvelopeBadPayload(t *testing.T) {
|
||||
c, _ := newTestClient(newFakeConn())
|
||||
c.sendEnvelope(Envelope{Type: TypeEcho, Payload: json.RawMessage("{")})
|
||||
c.sendEnvelope(Envelope{
|
||||
Type: TypeEcho,
|
||||
Payload: json.RawMessage("{"),
|
||||
})
|
||||
if len(c.send) != 0 {
|
||||
t.Errorf("queued %d messages; want none", len(c.send))
|
||||
}
|
||||
@@ -63,7 +70,9 @@ func TestSendEnvelopeSlowConsumer(t *testing.T) {
|
||||
f := newFakeConn()
|
||||
c, ctx := newTestClient(f)
|
||||
for range clientSendBuffer {
|
||||
c.sendEnvelope(Envelope{Type: TypePong})
|
||||
c.sendEnvelope(Envelope{
|
||||
Type: TypePong,
|
||||
})
|
||||
}
|
||||
select {
|
||||
case code := <-f.closed:
|
||||
@@ -71,7 +80,9 @@ func TestSendEnvelopeSlowConsumer(t *testing.T) {
|
||||
default:
|
||||
}
|
||||
|
||||
c.sendEnvelope(Envelope{Type: TypePong})
|
||||
c.sendEnvelope(Envelope{
|
||||
Type: TypePong,
|
||||
})
|
||||
select {
|
||||
case code := <-f.closed:
|
||||
if code != ws.StatusPolicyViolation {
|
||||
@@ -109,13 +120,31 @@ func TestClientMessageRateLimit(t *testing.T) {
|
||||
rule config.JSCRateLimitRule
|
||||
limited bool
|
||||
}{
|
||||
{config.JSCRateLimitRule{Enabled: false, PerSecond: 1, Burst: 1}, false},
|
||||
{config.JSCRateLimitRule{Enabled: true, PerSecond: 0, Burst: 1}, false},
|
||||
{config.JSCRateLimitRule{Enabled: true, PerSecond: 1, Burst: 1}, true},
|
||||
{config.JSCRateLimitRule{
|
||||
Enabled: false,
|
||||
PerSecond: 1,
|
||||
Burst: 1,
|
||||
}, false},
|
||||
{config.JSCRateLimitRule{
|
||||
Enabled: true,
|
||||
PerSecond: 0,
|
||||
Burst: 1,
|
||||
}, false},
|
||||
{config.JSCRateLimitRule{
|
||||
Enabled: true,
|
||||
PerSecond: 1,
|
||||
Burst: 1,
|
||||
}, true},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
h := NewHub(config.JSCServerWebSocket{MessageRateLimit: tt.rule}, WithLogger(discardLog))
|
||||
c := newClient(h, newFakeConn(), "", func() {})
|
||||
h := NewHub(
|
||||
config.JSCServerWebSocket{
|
||||
MessageRateLimit: tt.rule,
|
||||
},
|
||||
WithLogger(discardLog),
|
||||
)
|
||||
c := newClient(h, newFakeConn(), "", func() {
|
||||
})
|
||||
c.limiter.Allow()
|
||||
if limited := !c.limiter.Allow(); limited != tt.limited {
|
||||
t.Errorf("rule %+v: second message limited = %v; want %v", tt.rule, limited, tt.limited)
|
||||
@@ -133,7 +162,8 @@ func TestMustRaw(t *testing.T) {
|
||||
{"nil", nil, ""},
|
||||
{"raw message as is", raw, `{"a":1}`},
|
||||
{"marshalled", map[string]int{"b": 2}, `{"b":2}`},
|
||||
{"unmarshallable", func() {}, ""},
|
||||
{"unmarshallable", func() {
|
||||
}, ""},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
if got := string(mustRaw(tt.v)); got != tt.want {
|
||||
@@ -187,13 +217,20 @@ func (b *syncBuffer) entries(t *testing.T) []map[string]any {
|
||||
// buffer.
|
||||
func jsonLog(level slog.Level) (*slog.Logger, *syncBuffer) {
|
||||
buf := &syncBuffer{}
|
||||
return slog.New(slog.NewJSONHandler(buf, &slog.HandlerOptions{Level: level})), buf
|
||||
return slog.New(slog.NewJSONHandler(buf, &slog.HandlerOptions{
|
||||
Level: level,
|
||||
})), buf
|
||||
}
|
||||
|
||||
// At debug level, messages received from and sent to a client are logged.
|
||||
func TestLogMessages(t *testing.T) {
|
||||
log, buf := jsonLog(slog.LevelDebug)
|
||||
h := NewHub(config.JSCServerWebSocket{WriteTimeout: time.Second}, WithLogger(log))
|
||||
h := NewHub(
|
||||
config.JSCServerWebSocket{
|
||||
WriteTimeout: time.Second,
|
||||
},
|
||||
WithLogger(log),
|
||||
)
|
||||
peer, _ := pipeClient(t, h)
|
||||
|
||||
ping := `{"type":"ping","id":"1"}`
|
||||
@@ -236,7 +273,8 @@ func TestLogMessages(t *testing.T) {
|
||||
func TestLogMessageLimit(t *testing.T) {
|
||||
log, buf := jsonLog(slog.LevelDebug)
|
||||
h := NewHub(config.JSCServerWebSocket{}, WithLogger(log))
|
||||
c := newClient(h, newFakeConn(), "", func() {})
|
||||
c := newClient(h, newFakeConn(), "", func() {
|
||||
})
|
||||
long := strings.Repeat("x", logMessageLimit+100)
|
||||
c.logMessage(context.Background(), "message sent", []byte(long))
|
||||
|
||||
|
||||
@@ -30,16 +30,16 @@ func (h *Hub) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
ip := middleware.ClientIP(r)
|
||||
ctx, cancel := context.WithCancel(r.Context())
|
||||
c := newClient(h, conn, ip, cancel)
|
||||
client := newClient(h, conn, ip, cancel)
|
||||
|
||||
if !h.register(c) {
|
||||
if !h.register(client) {
|
||||
cancel()
|
||||
conn.Close(ws.StatusGoingAway, "server shutting down") //nolint:errcheck,gosec // turned away
|
||||
return
|
||||
}
|
||||
defer h.unregister(c)
|
||||
defer h.unregister(client)
|
||||
|
||||
h.logger().Info("client connected", "client", c.id, "ip", ip)
|
||||
c.run(ctx)
|
||||
h.logger().Info("client disconnected", "client", c.id, "ip", ip)
|
||||
h.logger().Info("client connected", "client", client.id, "ip", ip)
|
||||
client.run(ctx)
|
||||
h.logger().Info("client disconnected", "client", client.id, "ip", ip)
|
||||
}
|
||||
@@ -35,7 +35,9 @@ func dialWS(t *testing.T, ctx context.Context, url string) *ws.Conn {
|
||||
if err != nil {
|
||||
t.Fatalf("dial: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { conn.CloseNow() })
|
||||
t.Cleanup(func() {
|
||||
conn.CloseNow()
|
||||
})
|
||||
var e Envelope
|
||||
if err := wsjson.Read(ctx, conn, &e); err != nil || e.Type != TypeWelcome {
|
||||
t.Fatalf("want welcome, got %+v, %v", e, err)
|
||||
@@ -46,13 +48,21 @@ func dialWS(t *testing.T, ctx context.Context, url string) *ws.Conn {
|
||||
// A client upgraded over HTTP gets replies. A max_message_size of 0 uses the
|
||||
// default limit rather than refusing every message.
|
||||
func TestHTTPMessages(t *testing.T) {
|
||||
h := NewHub(config.JSCServerWebSocket{WriteTimeout: time.Second}, WithLogger(discardLog))
|
||||
h := NewHub(
|
||||
config.JSCServerWebSocket{
|
||||
WriteTimeout: time.Second,
|
||||
},
|
||||
WithLogger(discardLog),
|
||||
)
|
||||
url := wsHub(t, h)
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
|
||||
defer cancel()
|
||||
conn := dialWS(t, ctx, url)
|
||||
|
||||
if err := wsjson.Write(ctx, conn, Envelope{Type: TypePing, ID: "1"}); err != nil {
|
||||
if err := wsjson.Write(ctx, conn, Envelope{
|
||||
Type: TypePing,
|
||||
ID: "1",
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var e Envelope
|
||||
@@ -89,7 +99,8 @@ func TestPushBadPayload(t *testing.T) {
|
||||
c.read() // welcome
|
||||
waitCount(t, h, 1)
|
||||
|
||||
h.Push("bad", func() {})
|
||||
h.Push("bad", func() {
|
||||
})
|
||||
h.Push("good", nil)
|
||||
if e := c.read(); e.Type != "good" || e.Payload != nil {
|
||||
t.Fatalf("want good with no payload, got %+v", e)
|
||||
@@ -98,7 +109,12 @@ func TestPushBadPayload(t *testing.T) {
|
||||
|
||||
// A binary frame closes the connection with 1003 (unsupported data).
|
||||
func TestHTTPBinaryFrame(t *testing.T) {
|
||||
h := NewHub(config.JSCServerWebSocket{WriteTimeout: time.Second}, WithLogger(discardLog))
|
||||
h := NewHub(
|
||||
config.JSCServerWebSocket{
|
||||
WriteTimeout: time.Second,
|
||||
},
|
||||
WithLogger(discardLog),
|
||||
)
|
||||
url := wsHub(t, h)
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
|
||||
defer cancel()
|
||||
@@ -117,8 +133,12 @@ func TestHTTPBinaryFrame(t *testing.T) {
|
||||
// handled; the connection stays open.
|
||||
func TestHTTPMessageRateLimit(t *testing.T) {
|
||||
conf := config.JSCServerWebSocket{
|
||||
WriteTimeout: time.Second,
|
||||
MessageRateLimit: config.JSCRateLimitRule{Enabled: true, PerSecond: 1, Burst: 1},
|
||||
WriteTimeout: time.Second,
|
||||
MessageRateLimit: config.JSCRateLimitRule{
|
||||
Enabled: true,
|
||||
PerSecond: 1,
|
||||
Burst: 1,
|
||||
},
|
||||
}
|
||||
h := NewHub(conf, WithLogger(discardLog))
|
||||
url := wsHub(t, h)
|
||||
@@ -127,7 +147,10 @@ func TestHTTPMessageRateLimit(t *testing.T) {
|
||||
conn := dialWS(t, ctx, url)
|
||||
|
||||
for _, id := range []string{"1", "2"} {
|
||||
if err := wsjson.Write(ctx, conn, Envelope{Type: TypePing, ID: id}); err != nil {
|
||||
if err := wsjson.Write(ctx, conn, Envelope{
|
||||
Type: TypePing,
|
||||
ID: id,
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
+18
-16
@@ -17,7 +17,7 @@ type Hub struct {
|
||||
conf config.JSCServerWebSocket
|
||||
log *slog.Logger
|
||||
|
||||
m sync.RWMutex
|
||||
mu sync.RWMutex
|
||||
clients map[*client]struct{}
|
||||
closed bool
|
||||
wg sync.WaitGroup
|
||||
@@ -82,13 +82,13 @@ func (h *Hub) Push(msgType string, payload any) {
|
||||
}
|
||||
|
||||
var slow []*client
|
||||
h.m.RLock()
|
||||
h.mu.RLock()
|
||||
for client := range h.clients {
|
||||
if !client.trySend(msg) {
|
||||
slow = append(slow, client)
|
||||
}
|
||||
}
|
||||
h.m.RUnlock()
|
||||
h.mu.RUnlock()
|
||||
|
||||
for _, client := range slow {
|
||||
h.logger().Warn("dropping slow client", "client", client.id, "ip", client.ip)
|
||||
@@ -98,8 +98,8 @@ func (h *Hub) Push(msgType string, payload any) {
|
||||
|
||||
// Count returns the number of connected clients.
|
||||
func (h *Hub) Count() int {
|
||||
h.m.RLock()
|
||||
defer h.m.RUnlock()
|
||||
h.mu.RLock()
|
||||
defer h.mu.RUnlock()
|
||||
return len(h.clients)
|
||||
}
|
||||
|
||||
@@ -108,13 +108,13 @@ func (h *Hub) Count() int {
|
||||
// closed at once and ctx's error is returned. New connections are refused
|
||||
// afterwards.
|
||||
func (h *Hub) Shutdown(ctx context.Context) error {
|
||||
h.m.Lock()
|
||||
h.mu.Lock()
|
||||
h.closed = true
|
||||
clientShutdowns := make([]*client, 0, len(h.clients))
|
||||
for client := range h.clients {
|
||||
clientShutdowns = append(clientShutdowns, client)
|
||||
}
|
||||
h.m.Unlock()
|
||||
h.mu.Unlock()
|
||||
|
||||
if len(clientShutdowns) == 0 {
|
||||
// Nothing to close. Clients that just left are only finishing their
|
||||
@@ -151,7 +151,9 @@ func (h *Hub) Shutdown(ctx context.Context) error {
|
||||
}
|
||||
|
||||
// maxMessageSize is the largest message read from a client: max_message_size,
|
||||
// or defaultMaxMessageSize if that is 0 or less.
|
||||
// or defaultMaxMessageSize if that is 0 or less. The configuration rejects -1
|
||||
// (no limit) for this setting, as a client could then exhaust the server's
|
||||
// memory; a -1 set in code also gets the default, never no limit.
|
||||
func (h *Hub) maxMessageSize() int {
|
||||
if h.conf.MaxMessageSize <= 0 {
|
||||
return defaultMaxMessageSize
|
||||
@@ -161,21 +163,21 @@ func (h *Hub) maxMessageSize() int {
|
||||
|
||||
// register adds a client to the hub. Hub clients receive all push messages and
|
||||
// have their lifecycles tracked by the hub.
|
||||
func (h *Hub) register(c *client) bool {
|
||||
h.m.Lock()
|
||||
defer h.m.Unlock()
|
||||
func (h *Hub) register(client *client) bool {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
if h.closed {
|
||||
return false
|
||||
}
|
||||
h.clients[c] = struct{}{}
|
||||
h.clients[client] = struct{}{}
|
||||
h.wg.Add(1)
|
||||
return true
|
||||
}
|
||||
|
||||
// unregister removes a client from the hub.
|
||||
func (h *Hub) unregister(c *client) {
|
||||
h.m.Lock()
|
||||
delete(h.clients, c)
|
||||
h.m.Unlock()
|
||||
func (h *Hub) unregister(client *client) {
|
||||
h.mu.Lock()
|
||||
delete(h.clients, client)
|
||||
h.mu.Unlock()
|
||||
h.wg.Done()
|
||||
}
|
||||
@@ -17,9 +17,15 @@ import (
|
||||
func pipeClient(t *testing.T, h *Hub) (net.Conn, chan struct{}) {
|
||||
t.Helper()
|
||||
server, peer := net.Pipe()
|
||||
t.Cleanup(func() { server.Close(); peer.Close() })
|
||||
t.Cleanup(func() {
|
||||
server.Close()
|
||||
peer.Close()
|
||||
})
|
||||
done := make(chan struct{})
|
||||
go func() { h.ServeTCP(context.Background(), server); close(done) }()
|
||||
go func() {
|
||||
h.ServeTCP(context.Background(), server)
|
||||
close(done)
|
||||
}()
|
||||
peer.SetReadDeadline(time.Now().Add(time.Second))
|
||||
if _, err := bufio.NewReader(peer).ReadString('\n'); err != nil {
|
||||
t.Fatalf("welcome: %v", err)
|
||||
@@ -61,10 +67,13 @@ func TestShutdownNoClients(t *testing.T) {
|
||||
|
||||
// A ping that can't be sent within the pong timeout drops the client.
|
||||
func TestPingFailureDropsClient(t *testing.T) {
|
||||
h := NewHub(config.JSCServerWebSocket{
|
||||
PingInterval: 10 * time.Millisecond,
|
||||
PongTimeout: 20 * time.Millisecond,
|
||||
}, WithLogger(discardLog))
|
||||
h := NewHub(
|
||||
config.JSCServerWebSocket{
|
||||
PingInterval: 10 * time.Millisecond,
|
||||
PongTimeout: 20 * time.Millisecond,
|
||||
},
|
||||
WithLogger(discardLog),
|
||||
)
|
||||
_, done := pipeClient(t, h)
|
||||
|
||||
select {
|
||||
@@ -81,8 +90,12 @@ func TestMaxMessageSize(t *testing.T) {
|
||||
want int
|
||||
}{
|
||||
{config.JSCServerWebSocket{}, defaultMaxMessageSize},
|
||||
{config.JSCServerWebSocket{MaxMessageSize: -1}, defaultMaxMessageSize},
|
||||
{config.JSCServerWebSocket{MaxMessageSize: 100}, 100},
|
||||
{config.JSCServerWebSocket{
|
||||
MaxMessageSize: -1,
|
||||
}, defaultMaxMessageSize},
|
||||
{config.JSCServerWebSocket{
|
||||
MaxMessageSize: 100,
|
||||
}, 100},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
h := NewHub(tt.conf, WithLogger(discardLog))
|
||||
|
||||
@@ -37,28 +37,36 @@ type Envelope struct {
|
||||
// Envelope types. TypeEvent, TypeRequest and TypeAck are reserved: nothing
|
||||
// sends or handles them yet.
|
||||
const (
|
||||
// TypeWelcome is sent to a client when it connects, with its client ID.
|
||||
TypeWelcome = "welcome"
|
||||
TypePing = "ping"
|
||||
TypePong = "pong"
|
||||
TypeEcho = "echo"
|
||||
TypeEvent = "event"
|
||||
// TypePing asks the server for a TypePong.
|
||||
TypePing = "ping"
|
||||
// TypePong answers a TypePing, with the server's time.
|
||||
TypePong = "pong"
|
||||
// TypeEcho is sent back to the client with the same ID and payload.
|
||||
TypeEcho = "echo"
|
||||
// TypeEvent is reserved for events pushed to clients.
|
||||
TypeEvent = "event"
|
||||
// TypeRequest is reserved for requests from clients.
|
||||
TypeRequest = "request"
|
||||
TypeAck = "ack"
|
||||
TypeError = "error"
|
||||
// TypeAck is reserved for acknowledging a request.
|
||||
TypeAck = "ack"
|
||||
// TypeError reports an error, with the ID of the message it answers.
|
||||
TypeError = "error"
|
||||
)
|
||||
|
||||
// mustRaw returns v as raw JSON: v itself if it is already json.RawMessage,
|
||||
// nil if v is nil or can't be marshalled.
|
||||
func mustRaw(v any) json.RawMessage {
|
||||
if v == nil {
|
||||
// mustRaw returns value as raw JSON: value itself if it is already
|
||||
// json.RawMessage, nil if value is nil or can't be marshalled.
|
||||
func mustRaw(value any) json.RawMessage {
|
||||
if value == nil {
|
||||
return nil
|
||||
}
|
||||
if raw, ok := v.(json.RawMessage); ok {
|
||||
if raw, ok := value.(json.RawMessage); ok {
|
||||
return raw
|
||||
}
|
||||
b, err := json.Marshal(v)
|
||||
encoded, err := json.Marshal(value)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
return b
|
||||
return encoded
|
||||
}
|
||||
@@ -178,7 +178,10 @@ func (t *tcpConn) Ping(ctx context.Context) error {
|
||||
// connection. Only the first Close or CloseNow does anything; later calls
|
||||
// return nil. A CloseNow while Close is sending its message cuts it short.
|
||||
func (t *tcpConn) Close(code ws.StatusCode, reason string) error {
|
||||
if !t.markClosed(ws.CloseError{Code: code, Reason: reason}) {
|
||||
if !t.markClosed(ws.CloseError{
|
||||
Code: code,
|
||||
Reason: reason,
|
||||
}) {
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -206,7 +209,9 @@ func (t *tcpConn) Close(code ws.StatusCode, reason string) error {
|
||||
// CloseNow closes the connection without a message. It doesn't wait for a
|
||||
// Close that is sending its message.
|
||||
func (t *tcpConn) CloseNow() error {
|
||||
t.markClosed(ws.CloseError{Code: ws.StatusAbnormalClosure})
|
||||
t.markClosed(ws.CloseError{
|
||||
Code: ws.StatusAbnormalClosure,
|
||||
})
|
||||
if err := t.conn.Close(); err != nil && !errors.Is(err, net.ErrClosed) {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -69,8 +69,14 @@ func dialTCP(t *testing.T, addr string) *tcpClient {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { conn.Close() })
|
||||
return &tcpClient{t: t, conn: conn, r: bufio.NewReader(conn)}
|
||||
t.Cleanup(func() {
|
||||
conn.Close()
|
||||
})
|
||||
return &tcpClient{
|
||||
t: t,
|
||||
conn: conn,
|
||||
r: bufio.NewReader(conn),
|
||||
}
|
||||
}
|
||||
|
||||
func (c *tcpClient) send(line string) {
|
||||
@@ -155,7 +161,9 @@ func TestTCPMessages(t *testing.T) {
|
||||
// With no write timeout, messages are still sent: a zero timeout must not
|
||||
// mean one that has already expired.
|
||||
func TestTCPNoWriteTimeout(t *testing.T) {
|
||||
_, addr := tcpHub(t, func(c *config.JSCServerWebSocket) { c.WriteTimeout = 0 })
|
||||
_, addr := tcpHub(t, func(c *config.JSCServerWebSocket) {
|
||||
c.WriteTimeout = 0
|
||||
})
|
||||
c := dialTCP(t, addr)
|
||||
if e := c.read(); e.Type != TypeWelcome {
|
||||
t.Fatalf("want welcome, got %+v", e)
|
||||
@@ -177,7 +185,9 @@ func TestTCPClientDisconnect(t *testing.T) {
|
||||
|
||||
// A line longer than max_message_size drops the connection.
|
||||
func TestTCPLineTooLong(t *testing.T) {
|
||||
h, addr := tcpHub(t, func(c *config.JSCServerWebSocket) { c.MaxMessageSize = 64 })
|
||||
h, addr := tcpHub(t, func(c *config.JSCServerWebSocket) {
|
||||
c.MaxMessageSize = 64
|
||||
})
|
||||
c := dialTCP(t, addr)
|
||||
c.read() // welcome
|
||||
c.send(`{"type":"echo","payload":"` + strings.Repeat("x", 100) + `"}` + "\n")
|
||||
@@ -244,12 +254,21 @@ func TestTCPPing(t *testing.T) {
|
||||
// full. net.Pipe has no buffer, so writes to it block until the peer reads.
|
||||
func TestTCPSlowConsumer(t *testing.T) {
|
||||
h := NewHub(
|
||||
config.JSCServerWebSocket{WriteTimeout: 50 * time.Millisecond}, WithLogger(discardLog),
|
||||
config.JSCServerWebSocket{
|
||||
WriteTimeout: 50 * time.Millisecond,
|
||||
},
|
||||
WithLogger(discardLog),
|
||||
)
|
||||
server, peer := net.Pipe()
|
||||
t.Cleanup(func() { server.Close(); peer.Close() })
|
||||
t.Cleanup(func() {
|
||||
server.Close()
|
||||
peer.Close()
|
||||
})
|
||||
done := make(chan struct{})
|
||||
go func() { h.ServeTCP(context.Background(), server); close(done) }()
|
||||
go func() {
|
||||
h.ServeTCP(context.Background(), server)
|
||||
close(done)
|
||||
}()
|
||||
waitCount(t, h, 1)
|
||||
|
||||
for range clientSendBuffer + 1 {
|
||||
@@ -283,7 +302,9 @@ func TestTCPNoPongTimeout(t *testing.T) {
|
||||
|
||||
// A max_message_size of 0 uses the default limit.
|
||||
func TestTCPDefaultMessageSize(t *testing.T) {
|
||||
_, addr := tcpHub(t, func(c *config.JSCServerWebSocket) { c.MaxMessageSize = 0 })
|
||||
_, addr := tcpHub(t, func(c *config.JSCServerWebSocket) {
|
||||
c.MaxMessageSize = 0
|
||||
})
|
||||
c := dialTCP(t, addr)
|
||||
c.read() // welcome
|
||||
c.send(`{"type":"echo","id":"1","payload":"` + strings.Repeat("x", 4096) + `"}` + "\n")
|
||||
@@ -299,7 +320,10 @@ func TestTCPDefaultMessageSize(t *testing.T) {
|
||||
func pipeConn(t *testing.T, closeTimeout time.Duration) (*tcpConn, net.Conn) {
|
||||
t.Helper()
|
||||
server, peer := net.Pipe()
|
||||
t.Cleanup(func() { server.Close(); peer.Close() })
|
||||
t.Cleanup(func() {
|
||||
server.Close()
|
||||
peer.Close()
|
||||
})
|
||||
return newTCPConn(server, 0, closeTimeout), peer
|
||||
}
|
||||
|
||||
@@ -370,13 +394,17 @@ func TestTCPConnClose(t *testing.T) {
|
||||
func TestTCPConnCloseNowDuringClose(t *testing.T) {
|
||||
tc, _ := pipeConn(t, 0) // no timeout: Close would wait for ever
|
||||
closed := make(chan error, 1)
|
||||
go func() { closed <- tc.Close(ws.StatusGoingAway, "bye") }()
|
||||
go func() {
|
||||
closed <- tc.Close(ws.StatusGoingAway, "bye")
|
||||
}()
|
||||
for !tc.isClosed() {
|
||||
time.Sleep(time.Millisecond)
|
||||
}
|
||||
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- tc.CloseNow() }()
|
||||
go func() {
|
||||
done <- tc.CloseNow()
|
||||
}()
|
||||
select {
|
||||
case err := <-done:
|
||||
if err != nil {
|
||||
@@ -396,7 +424,9 @@ func TestTCPConnCloseNowDuringClose(t *testing.T) {
|
||||
func TestTCPConnCloseTimeout(t *testing.T) {
|
||||
tc, _ := pipeConn(t, 20*time.Millisecond)
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- tc.Close(ws.StatusGoingAway, "bye") }()
|
||||
go func() {
|
||||
done <- tc.Close(ws.StatusGoingAway, "bye")
|
||||
}()
|
||||
select {
|
||||
case err := <-done:
|
||||
if err != nil {
|
||||
|
||||
@@ -23,10 +23,14 @@ var discardLog = slog.New(slog.NewTextHandler(io.Discard, nil))
|
||||
// not nil.
|
||||
func testConf(mutate func(*config.JSCServer)) *config.JSCServer {
|
||||
c := &config.JSCServer{
|
||||
Host: "127.0.0.1",
|
||||
Port: 0,
|
||||
WebSocket: config.JSCServerWebSocket{WriteTimeout: time.Second},
|
||||
Timeouts: config.JSCServerTimeouts{ShutdownTimeout: time.Second},
|
||||
Host: "127.0.0.1",
|
||||
Port: 0,
|
||||
WebSocket: config.JSCServerWebSocket{
|
||||
WriteTimeout: time.Second,
|
||||
},
|
||||
Timeouts: config.JSCServerTimeouts{
|
||||
ShutdownTimeout: time.Second,
|
||||
},
|
||||
}
|
||||
if mutate != nil {
|
||||
mutate(c)
|
||||
@@ -54,7 +58,9 @@ func startServer(
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
t.Cleanup(cancel)
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- s.serve(ctx, ln) }()
|
||||
go func() {
|
||||
done <- s.serve(ctx, ln)
|
||||
}()
|
||||
return h, ln.Addr().String(), cancel, done
|
||||
}
|
||||
|
||||
@@ -71,8 +77,14 @@ func dial(t *testing.T, addr string) *tcpClient {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { conn.Close() })
|
||||
return &tcpClient{t: t, conn: conn, r: bufio.NewReader(conn)}
|
||||
t.Cleanup(func() {
|
||||
conn.Close()
|
||||
})
|
||||
return &tcpClient{
|
||||
t: t,
|
||||
conn: conn,
|
||||
r: bufio.NewReader(conn),
|
||||
}
|
||||
}
|
||||
|
||||
// read returns the next envelope, failing the test after a second.
|
||||
@@ -188,7 +200,9 @@ func TestServerRun(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestServerRunListenError(t *testing.T) {
|
||||
s, _ := newServer(testConf(func(c *config.JSCServer) { c.Port = -1 }))
|
||||
s, _ := newServer(testConf(func(c *config.JSCServer) {
|
||||
c.Port = -1
|
||||
}))
|
||||
if err := s.Run(context.Background()); err == nil || !strings.Contains(err.Error(), "listen") {
|
||||
t.Fatalf("Run = %v; want a listen error", err)
|
||||
}
|
||||
@@ -217,7 +231,10 @@ func TestServerAcceptRetries(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer ln.Close()
|
||||
fl := &flakyListener{Listener: ln, fails: 3}
|
||||
fl := &flakyListener{
|
||||
Listener: ln,
|
||||
fails: 3,
|
||||
}
|
||||
err = s.accept(context.Background(), context.Background(), fl)
|
||||
if !errors.Is(err, net.ErrClosed) || fl.fails != 0 {
|
||||
t.Fatalf("accept = %v with %d failures left; want net.ErrClosed", err, fl.fails)
|
||||
@@ -233,7 +250,10 @@ func TestServerAcceptCancelledInBackOff(t *testing.T) {
|
||||
}
|
||||
defer ln.Close()
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
fl := &flakyListener{Listener: ln, fails: 1 << 30}
|
||||
fl := &flakyListener{
|
||||
Listener: ln,
|
||||
fails: 1 << 30,
|
||||
}
|
||||
time.AfterFunc(20*time.Millisecond, cancel)
|
||||
if err := s.accept(ctx, ctx, fl); !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("accept = %v; want context.Canceled", err)
|
||||
|
||||
Reference in new issue
Block a user