Added websocket hub for handling websocket clients, with a wsserve command

which runs a plain TCP server for receiving websocket hub clients, with
server config, CSRF and rate limit middleware.

- internal/websocket: Hub pushing messages to WebSocket clients (ServeHTTP,
  example.com/ws upgrade with optional CSRF check) and plain TCP clients (ServeTCP,
  newline-delimited JSON envelopes); ping/pong keepalive, per-client message
  rate limit, slow consumer handling, graceful Shutdown, debug logging of
  messages sent and received
- internal/websocket/tcpserver: standalone TCP server for a hub, with accept
  back-off, a per-IP connection limit (server.websocket.connection_limit) and
  graceful shutdown
- internal/cli/wsserve: `wsserve` command running the TCP server, with
  --listen/--host/--port overrides
- internal/middleware: keyed token bucket rate limiter, real client IP from
  trusted proxies
- internal/httpx: RFC 9457 problem responses, request IDs in context
- internal/config: server section (host, port, shutdown timeout, rate limiter
  idle TTL, websocket); util.ByteSize settings (e.g. "32KiB"); lists of
  sections as single keys; empty lists stored as nil; isExplicitlySet for
  printLoadedConfig; JobSearchConfigSource renamed to JSCSourcesSource;
  TestSections checks every section's methods
- internal/util: ByteSize, parsed and formatted with go-humanize

internal/config/README.md

@@ -40,11 +40,26 @@ is kept.
 | `sources.find_a_tender.api_root` | string | `https://www.find-tender.service.gov.uk/api/1.0` | URL of the Find a Tender API |
 | `sources.fts_mock.api_root` | string | | URL of the mock FTS server (see `internal/mock`) |
 | `sources.sleeper.block_shutdown` | bool | `false` | Test source: if true, its sleep ignores shutdown |
+| `server.host` | string | `127.0.0.1` | Address to listen on |
+| `server.port` | int | `9393` | Port to listen on |
+| `server.timeouts.shutdown` | duration | `15s` | How long shutdown waits for clients to close |
+| `server.rate_limits.idle_ttl` | duration | `10m` | How long a client's rate limiter is kept after its last request |
+| `server.websocket.enable_compression` | bool | `false` | permessage-deflate |
+| `server.websocket.max_message_size` | byte size | `32KiB` | Largest message read from a client; `0B` for 32KiB |
+| `server.websocket.connection_limit.*` | | | A rate limit rule for new connections from each client IP, WebSocket and plain TCP alike; limiters are kept for `server.rate_limits.idle_ttl` |
+| `server.websocket.message_rate_limit.*` | | | A rate limit rule for messages from each connection |
+| `server.websocket.write_timeout` | duration | `10s` | `0s` for none |
+| `server.websocket.ping_interval` | duration | `30s` | |
+| `server.websocket.pong_timeout` | duration | `10s` | `0s` for none |
 
 The current sources are `find_a_tender`, `fts_mock` and `sleeper`. See 
 "Addition of a new opportunities source" in the top level README for how to add
 a new source.
 
+A rate limit rule (`JSCRateLimitRule`) has `enabled`, `per_second` and `burst`.
+It is used under `server.websocket.connection_limit` and
+`server.websocket.message_rate_limit`.
+
 Example `config.yaml`:
 ```yaml
 logging:
@@ -79,6 +94,9 @@ per key:
   `time.Duration` or a duration string in `SetValue`, and are given out as
   strings (`"10m0s"`) by `Map`, `Flatten` and `ValueOf`, and so by `JSON`,
   `YAML` and `String` too.
+- `util.ByteSize` fields work the same way, with size strings such as `"32KiB"`
+  or `"1MB"` (KB, MB, ... are powers of 1000; KiB, MiB, ... powers of 1024),
+  and are given out exactly in the largest unit that divides them (`"1MB"`).
 - A section can reject values by implementing `validate(key, value) error` on
   its pointer type. `JobSearchConfigLogging` does this for `level`.
 
@@ -138,6 +156,9 @@ Nothing in `section.go` needs changing.
 | `conf_logging.go` | `logging` section and log level helpers |
 | `conf_sources.go` | `sources` section and the shared source fields |
 | `conf_source_*.go` | One file per source section |
+| `conf_server.go` | `server` section |
+| `conf_server_*.go` | One file per `server` sub-section (`rate_limits`, `timeouts`, `websocket`) |
+| `conf_ratelimitrule.go` | `JSCRateLimitRule`, the rate limit rule shared by several keys |
 | `print.go`, `debug.go` | Printing and debug logging of the configuration |
 
 

internal/config/conf_ratelimitrule.go

@@ -0,0 +1,29 @@
+package config
+
+import "reflect"
+
+// JSCRateLimitRule configures one rate limit. The type is used under several
+// keys so it has no JSCK* constant and its Flatten keys are not prefixed.
+type JSCRateLimitRule struct {
+	Enabled   bool `cfg:"enabled"             default:"false"`
+	PerSecond int  `cfg:"per_second" default:"50"`
+	Burst     int  `cfg:"burst"               default:"100"`
+}
+
+// ---------- Section methods (see Section) ----------
+
+func (c JSCRateLimitRule) Map() map[string]any { return mapOf(c) }
+
+func (c JSCRateLimitRule) Flatten() map[string]any { return flattenOf("", c) }
+
+func (c JSCRateLimitRule) TypeOf(key string) (reflect.Kind, error) {
+	return typeOf(c, key)
+}
+
+func (c JSCRateLimitRule) ValueOf(key string) (any, error) {
+	return valueOf(c, key)
+}
+
+func (c *JSCRateLimitRule) SetValue(key string, value any) error {
+	return setValue(c, key, value)
+}

internal/config/conf_sections_test.go

@@ -50,10 +50,10 @@ func TestDefaults(t *testing.T) {
 	if conf.Sources.FTSMock.APIRoot != "" {
 		t.Errorf("FTSMock.APIRoot = %q; want empty", conf.Sources.FTSMock.APIRoot)
 	}
-	for name, src := range map[string]JobSearchConfigSource{
-		"find_a_tender": conf.Sources.FindATender.JobSearchConfigSource,
-		"fts_mock":      conf.Sources.FTSMock.JobSearchConfigSource,
-		"sleeper":       conf.Sources.Sleeper.JobSearchConfigSource,
+	for name, src := range map[string]JSCSourcesSource{
+		"find_a_tender": conf.Sources.FindATender.JSCSourcesSource,
+		"fts_mock":      conf.Sources.FTSMock.JSCSourcesSource,
+		"sleeper":       conf.Sources.Sleeper.JSCSourcesSource,
 	} {
 		if src.Enabled || src.Interval != 10*time.Minute {
 			t.Errorf("%s source = %+v; want disabled, 10m", name, src)
@@ -150,7 +150,7 @@ func TestConfigLevelToLogLevel(t *testing.T) {
 }
 
 func TestSourceSection(t *testing.T) {
-	c := JobSearchConfigSource{Enabled: true, Interval: time.Minute}
+	c := JSCSourcesSource{Enabled: true, Interval: time.Minute}
 
 	if got, want := c.Keys(), []string{"enabled", "interval"}; !reflect.DeepEqual(got, want) {
 		t.Errorf("Keys = %v; want %v", got, want)
@@ -352,6 +352,7 @@ func TestSerialisedRoundTrip(t *testing.T) {
 	orig.Logging.Level = "info"
 	orig.Sources.FTSMock.Enabled = true
 	orig.Sources.FTSMock.APIRoot = "http://mock"
+	orig.Server.WebSocket.MaxMessageSize = 64 << 10
 	for _, format := range []string{"json", "yaml"} {
 		t.Run(format, func(t *testing.T) {
 			resetViper(t)

internal/config/conf_server.go

@@ -0,0 +1,37 @@
+package config
+
+import (
+	"reflect"
+)
+
+// JSCKServer is the server section's configuration key.
+const JSCKServer string = "server"
+
+// JSCServer configures the outward server.
+type JSCServer struct {
+	Host       string              `cfg:"host"        default:"127.0.0.1"`
+	Port       int                 `cfg:"port"        default:"9393"`
+	Timeouts   JSCServerTimeouts   `cfg:"timeouts"`
+	RateLimits JSCServerRateLimits `cfg:"rate_limits"`
+	WebSocket  JSCServerWebSocket  `cfg:"websocket"`
+}
+
+// ---------- Section methods (see Section) ----------
+
+func (c JSCServer) Map() map[string]any { return mapOf(c) }
+
+func (c JSCServer) Flatten() map[string]any {
+	return flattenOf(JSCKServer, c)
+}
+
+func (c JSCServer) TypeOf(key string) (reflect.Kind, error) {
+	return typeOf(c, key)
+}
+
+func (c JSCServer) ValueOf(key string) (any, error) {
+	return valueOf(c, key)
+}
+
+func (c *JSCServer) SetValue(key string, value any) error {
+	return setValue(c, key, value)
+}

internal/config/conf_server_ratelimits.go

@@ -0,0 +1,35 @@
+package config
+
+import (
+	"reflect"
+	"time"
+)
+
+// JSCKServerRateLimits is the server rate limits section's configuration key.
+const JSCKServerRateLimits string = "rate_limits"
+
+// JSCServerRateLimits configures the server's rate limiters. IdleTTL is how
+// long a client's limiter is kept after its last request.
+type JSCServerRateLimits struct {
+	IdleTTL time.Duration `cfg:"idle_ttl" default:"10m"`
+}
+
+// ---------- Section methods (see Section) ----------
+
+func (c JSCServerRateLimits) Map() map[string]any { return mapOf(c) }
+
+func (c JSCServerRateLimits) Flatten() map[string]any {
+	return flattenOf(JSCKServerRateLimits, c)
+}
+
+func (c JSCServerRateLimits) TypeOf(key string) (reflect.Kind, error) {
+	return typeOf(c, key)
+}
+
+func (c JSCServerRateLimits) ValueOf(key string) (any, error) {
+	return valueOf(c, key)
+}
+
+func (c *JSCServerRateLimits) SetValue(key string, value any) error {
+	return setValue(c, key, value)
+}

internal/config/conf_server_timeouts.go

@@ -0,0 +1,35 @@
+package config
+
+import (
+	"reflect"
+	"time"
+)
+
+// JSCKServerTimeouts is the server timeouts section's configuration key.
+const JSCKServerTimeouts string = "timeouts"
+
+// JSCServerTimeouts configures the server's timeouts. ShutdownTimeout is how
+// long shutdown waits for clients to close.
+type JSCServerTimeouts struct {
+	ShutdownTimeout time.Duration `cfg:"shutdown" default:"15s"`
+}
+
+// ---------- Section methods (see Section) ----------
+
+func (c JSCServerTimeouts) Map() map[string]any { return mapOf(c) }
+
+func (c JSCServerTimeouts) Flatten() map[string]any {
+	return flattenOf(JSCKServerTimeouts, c)
+}
+
+func (c JSCServerTimeouts) TypeOf(key string) (reflect.Kind, error) {
+	return typeOf(c, key)
+}
+
+func (c JSCServerTimeouts) ValueOf(key string) (any, error) {
+	return valueOf(c, key)
+}
+
+func (c *JSCServerTimeouts) SetValue(key string, value any) error {
+	return setValue(c, key, value)
+}

internal/config/conf_server_websocket.go

@@ -0,0 +1,42 @@
+package config
+
+import (
+	"reflect"
+	"time"
+
+	"git.avsdev.uk/Go/job-search/internal/util"
+)
+
+// JSCKServerWebSocket is the websocket section's configuration key.
+const JSCKServerWebSocket string = "websocket"
+
+// JSCServerWebSocket configures the server's websocket connections.
+type JSCServerWebSocket struct {
+	EnableCompression bool             `cfg:"enable_compression" default:"false"`
+	MaxMessageSize    util.ByteSize    `cfg:"max_message_size"   default:"32KiB"`
+	ConnectionLimit   JSCRateLimitRule `cfg:"connection_limit"`
+	MessageRateLimit  JSCRateLimitRule `cfg:"message_rate_limit"`
+	WriteTimeout      time.Duration    `cfg:"write_timeout"      default:"10s"`
+	PingInterval      time.Duration    `cfg:"ping_interval"      default:"30s"`
+	PongTimeout       time.Duration    `cfg:"pong_timeout"       default:"10s"`
+}
+
+// ---------- Section methods (see Section) ----------
+
+func (c JSCServerWebSocket) Map() map[string]any { return mapOf(c) }
+
+func (c JSCServerWebSocket) Flatten() map[string]any {
+	return flattenOf(JSCKServerWebSocket, c)
+}
+
+func (c JSCServerWebSocket) TypeOf(key string) (reflect.Kind, error) {
+	return typeOf(c, key)
+}
+
+func (c JSCServerWebSocket) ValueOf(key string) (any, error) {
+	return valueOf(c, key)
+}
+
+func (c *JSCServerWebSocket) SetValue(key string, value any) error {
+	return setValue(c, key, value)
+}

internal/config/conf_source_findatender.go

@@ -8,7 +8,7 @@ const JSCKSourceFindATender string = "find_a_tender"
 // JSCSourceFindATender configures fetching from the Find a Tender
 // OCDS API at APIRoot.
 type JSCSourceFindATender struct {
-	JobSearchConfigSource
+	JSCSourcesSource
 	APIRoot string `cfg:"api_root" default:"https://www.find-tender.service.gov.uk/api/1.0"`
 }
 

internal/config/conf_source_ftsmock.go

@@ -8,7 +8,7 @@ const JSCKSourceFTSMock string = "fts_mock"
 // JSCSourceFTSMock configures fetching from a Find a Tender mock
 // server (cmd/mock-fts) at APIRoot.
 type JSCSourceFTSMock struct {
-	JobSearchConfigSource
+	JSCSourcesSource
 	APIRoot string `cfg:"api_root"`
 }
 

internal/config/conf_source_sleeper.go

@@ -9,7 +9,7 @@ const JSCKSourceSleeper string = "sleeper"
 // simulates work by sleeping. With BlockShutdown its work ignores
 // cancellation, to test shutdown waiting on a non-cooperative task.
 type JSCSourceSleeper struct {
-	JobSearchConfigSource
+	JSCSourcesSource
 	BlockShutdown bool `cfg:"block_shutdown"`
 }
 

internal/config/conf_sources.go

@@ -38,32 +38,32 @@ func (c *JSCSources) SetValue(key string, value any) error {
 
 // ========== Generic source ==========
 
-// JobSearchConfigSource holds the settings every source has, and is embedded
+// JSCSourcesSource holds the settings every source has, and is embedded
 // in each source's section. Interval is how often an enabled source fetches.
-type JobSearchConfigSource struct {
+type JSCSourcesSource struct {
 	Enabled  bool          `cfg:"enabled"`
 	Interval time.Duration `cfg:"interval" default:"10m"`
 }
 
 // ---------- Section methods (see Section) ----------
 
-func (c JobSearchConfigSource) Map() map[string]any { return mapOf(c) }
+func (c JSCSourcesSource) Map() map[string]any { return mapOf(c) }
 
-func (c JobSearchConfigSource) Flatten() map[string]any {
+func (c JSCSourcesSource) Flatten() map[string]any {
 	return flattenOf("", c)
 }
 
-func (c JobSearchConfigSource) TypeOf(key string) (reflect.Kind, error) {
+func (c JSCSourcesSource) TypeOf(key string) (reflect.Kind, error) {
 	return typeOf(c, key)
 }
 
-func (c JobSearchConfigSource) ValueOf(key string) (any, error) {
+func (c JSCSourcesSource) ValueOf(key string) (any, error) {
 	return valueOf(c, key)
 }
 
-func (c *JobSearchConfigSource) SetValue(key string, value any) error {
+func (c *JSCSourcesSource) SetValue(key string, value any) error {
 	return setValue(c, key, value)
 }
 
 // Keys returns the keys directly in the section.
-func (c *JobSearchConfigSource) Keys() []string { return keysOf(c) }
+func (c *JSCSourcesSource) Keys() []string { return keysOf(c) }

internal/config/config.go

@@ -24,9 +24,10 @@ const EtcConfigPath string = "/etc/job-search"
 // JobSearchConfig is the application's whole configuration, one field per
 // section. Each field's cfg tag must match the section's JSCK* constant.
 type JobSearchConfig struct {
-	Database  JSCDatabase        `cfg:"database"`
-	Logging   JSCLogging         `cfg:"logging"`
-	Sources   JSCSources         `cfg:"sources"`
+	Database JSCDatabase `cfg:"database"`
+	Logging  JSCLogging  `cfg:"logging"`
+	Server   JSCServer   `cfg:"server"`
+	Sources  JSCSources  `cfg:"sources"`
 }
 
 // LoadDefaultConfig sets Viper's defaults from the configuration's default
@@ -88,6 +89,24 @@ func viperValue(key string, kind reflect.Kind) (value any, ok bool) {
 		return viper.GetInt(key), true
 	case reflect.Bool:
 		return viper.GetBool(key), true
+	case reflect.Slice:
+		// A string, e.g. from an environment variable, is passed on for
+		// SetValue to split on commas; GetStringSlice would split it on
+		// spaces. A list of objects (a list of sections) is passed on as it
+		// is, as GetStringSlice would flatten it.
+		switch v := viper.Get(key).(type) {
+		case string:
+			return v, true
+		case []map[string]any: // as LoadToViper sets it
+			return v, true
+		case []any:
+			if len(v) > 0 {
+				if _, ok := v[0].(map[string]any); ok {
+					return v, true
+				}
+			}
+		}
+		return viper.GetStringSlice(key), true
 	}
 	return nil, false
 }

internal/config/config_test.go

@@ -1,13 +1,16 @@
 package config
 
 import (
+	"fmt"
+	"maps"
 	"reflect"
+	"strings"
 	"testing"
 )
 
 // Kinds that LoadFromViper knows how to read.
 var viperKinds = map[reflect.Kind]bool{
-	reflect.String: true, reflect.Int: true, reflect.Bool: true,
+	reflect.String: true, reflect.Int: true, reflect.Bool: true, reflect.Slice: true,
 }
 
 // Every key round-trips through TypeOf/ValueOf/SetValue and has a type that
@@ -39,19 +42,40 @@ var sectionType = reflect.TypeOf((*Section)(nil)).Elem()
 
 // Every section, including embedded ones, implements Section, and its
 // Flatten keys (which use the section's JSCK* constant) line up with its cfg
-// tag in the parent. Sections are found by walking JobSearchConfig, so a new
-// section is checked without being listed anywhere.
+// tag in the parent. A section type used under more than one key anywhere in
+// the config, such as JSCRateLimitRule, has no constant, so its keys are
+// unprefixed. Each section's Map, TypeOf, ValueOf and SetValue agree with the
+// whole configuration's view of the same keys, and so do those of each item
+// in a list of sections. Sections are found by walking JobSearchConfig, so a
+// new section is checked without being listed anywhere.
 func TestSections(t *testing.T) {
-	var conf JobSearchConfig
+	conf := defaultConfig(t)
 	if !reflect.PointerTo(reflect.TypeOf(conf)).Implements(sectionType) {
 		t.Error("*JobSearchConfig does not implement Section")
 	}
 	all := conf.Flatten()
-	checkSections(t, all, "", reflect.ValueOf(&conf).Elem())
+	uses := map[reflect.Type]int{}
+	countSections(uses, reflect.TypeOf(conf))
+	checkSections(t, &conf, all, uses, "", reflect.ValueOf(&conf).Elem())
 }
 
-func checkSections(t *testing.T, all map[string]any, prefix string, section reflect.Value) {
+// countSections counts how many fields of each section type there are in the
+// tree under section.
+func countSections(uses map[reflect.Type]int, section reflect.Type) {
+	for _, f := range fieldsOf(section) {
+		if f.section {
+			uses[f.typ]++
+			countSections(uses, f.typ)
+		}
+	}
+}
+
+func checkSections(t *testing.T, conf *JobSearchConfig, all map[string]any, uses map[reflect.Type]int, prefix string, section reflect.Value) {
 	for _, f := range fieldsOf(section.Type()) {
+		key := joinKey(prefix, f.key)
+		if isSectionList(f.typ) {
+			checkSectionList(t, all, key, section.FieldByIndex(f.index))
+		}
 		if !f.section {
 			continue
 		}
@@ -59,23 +83,106 @@ func checkSections(t *testing.T, all map[string]any, prefix string, section refl
 		name := fv.Type().Name()
 		if !fv.Addr().Type().Implements(sectionType) {
 			t.Errorf("*%s does not implement Section", name)
+			continue
 		}
 		checkEmbedded(t, fv.Type())
-		if flatten := fv.MethodByName("Flatten"); flatten.IsValid() {
-			keys := flatten.Call(nil)[0].Interface().(map[string]any)
-			for k := range keys {
-				if _, ok := all[joinKey(prefix, k)]; !ok {
-					t.Errorf("%s.Flatten() key %q does not match its cfg tag %q",
-						name, k, joinKey(prefix, f.key))
-				}
+		keys := fv.Addr().Interface().(Section).Flatten()
+		keyPrefix := prefix
+		if uses[f.typ] > 1 {
+			keyPrefix = key
+		}
+		for k := range keys {
+			if _, ok := all[joinKey(keyPrefix, k)]; !ok {
+				t.Errorf("%s.Flatten() key %q does not match its cfg tag %q",
+					name, k, key)
 			}
 		}
-		checkSections(t, all, joinKey(prefix, f.key), fv)
+
+		// The section's own keys, relative to it, with the whole
+		// configuration's values and kinds for them.
+		want := map[string]any{}
+		kinds := map[string]reflect.Kind{}
+		for k, v := range all {
+			if rel, ok := strings.CutPrefix(k, key+"."); ok {
+				want[rel] = v
+				kinds[rel], _ = conf.TypeOf(k)
+			}
+		}
+		checkSectionMethods(t, name, fv, want, kinds)
+		checkSections(t, conf, all, uses, key, fv)
 	}
 }
 
+// checkSectionList checks the items of the list of sections at key implement
+// Section and that their methods agree with the list's value in all.
+func checkSectionList(t *testing.T, all map[string]any, key string, list reflect.Value) {
+	name := list.Type().Elem().Name()
+	if !reflect.PointerTo(list.Type().Elem()).Implements(sectionType) {
+		t.Errorf("*%s (items of %s) does not implement Section", name, key)
+		return
+	}
+	items, _ := all[key].([]map[string]any)
+	if len(items) != list.Len() {
+		t.Errorf("%s has %d items; Flatten gives %d", key, list.Len(), len(items))
+		return
+	}
+	for i := range list.Len() {
+		item, itemName := list.Index(i), fmt.Sprintf("%s[%d]", key, i)
+		want := flattenMap("", items[i])
+		// Items are not prefixed, and their kinds are their fields' kinds.
+		if got := item.Addr().Interface().(Section).Flatten(); !reflect.DeepEqual(got, want) {
+			t.Errorf("%s.Flatten() =\n%v\nwant\n%v", itemName, got, want)
+		}
+		kinds := map[string]reflect.Kind{}
+		for k := range want {
+			_, f, _ := lookup(item, k)
+			kinds[k] = kindOf(f.typ)
+		}
+		checkSectionMethods(t, itemName, item, want, kinds)
+	}
+}
+
+// checkSectionMethods checks section's Map, TypeOf, ValueOf and SetValue
+// against want, its keys and values, and kinds, their kinds.
+// Setting each key to its own value must succeed and leave section unchanged.
+func checkSectionMethods(t *testing.T, name string, section reflect.Value, want map[string]any, kinds map[string]reflect.Kind) {
+	s := section.Addr().Interface().(Section)
+	if got := flattenMap("", s.Map()); !reflect.DeepEqual(got, want) {
+		t.Errorf("%s.Map() =\n%v\nwant\n%v", name, got, want)
+	}
+	before := section.Interface()
+	for k, v := range want {
+		if kind, err := s.TypeOf(k); err != nil || kind != kinds[k] {
+			t.Errorf("%s.TypeOf(%q) = %v, %v; want %v", name, k, kind, err, kinds[k])
+		}
+		if got, err := s.ValueOf(k); err != nil || !reflect.DeepEqual(got, v) {
+			t.Errorf("%s.ValueOf(%q) = %#v, %v; want %#v", name, k, got, err, v)
+		}
+		if err := s.SetValue(k, v); err != nil {
+			t.Errorf("%s.SetValue(%q, %#v): %v", name, k, v, err)
+		}
+	}
+	if !reflect.DeepEqual(section.Interface(), before) {
+		t.Errorf("%s changed by setting its keys to their own values", name)
+	}
+}
+
+// flattenMap flattens a map with nested maps, as Map gives, into dotted keys
+// prefixed with prefix (if non-empty).
+func flattenMap(prefix string, m map[string]any) map[string]any {
+	flat := map[string]any{}
+	for k, v := range m {
+		if nested, ok := v.(map[string]any); ok {
+			maps.Copy(flat, flattenMap(joinKey(prefix, k), nested))
+		} else {
+			flat[joinKey(prefix, k)] = v
+		}
+	}
+	return flat
+}
+
 // checkEmbedded checks that structs embedded in a section, such as
-// JobSearchConfigSource, implement Section too.
+// JSCSourcesSource, implement Section too.
 func checkEmbedded(t *testing.T, section reflect.Type) {
 	for i := 0; i < section.NumField(); i++ {
 		sf := section.Field(i)

internal/config/modify.go

@@ -73,6 +73,8 @@ func parseValue(kind reflect.Kind, value string) (any, error) {
 		return strconv.ParseUint(value, 10, 64)
 	case reflect.Float32, reflect.Float64:
 		return strconv.ParseFloat(value, 64)
+	case reflect.Slice:
+		return splitList(value), nil
 	}
 	return nil, errors.ErrUnsupported
 }

internal/config/modify_test.go

@@ -181,9 +181,10 @@ func TestParseValue(t *testing.T) {
 		{reflect.Int8, "7", int64(7)},
 		{reflect.Uint16, "65535", uint64(65535)},
 		{reflect.Float32, "1.5", 1.5},
+		{reflect.Slice, " a, b ,, c ", []string{"a", "b", "c"}},
 	} {
 		got, err := parseValue(tt.kind, tt.value)
-		if err != nil || got != tt.want {
+		if err != nil || !reflect.DeepEqual(got, tt.want) {
 			t.Errorf("parseValue(%v, %q) = %#v, %v; want %#v", tt.kind, tt.value, got, err, tt.want)
 		}
 	}
@@ -202,8 +203,8 @@ func TestParseValue(t *testing.T) {
 		}
 	}
 
-	if _, err := parseValue(reflect.Slice, "a,b"); !errors.Is(err, errors.ErrUnsupported) {
-		t.Errorf("parseValue(slice) err = %v; want ErrUnsupported", err)
+	if _, err := parseValue(reflect.Map, "a"); !errors.Is(err, errors.ErrUnsupported) {
+		t.Errorf("parseValue(map) err = %v; want ErrUnsupported", err)
 	}
 }
 

internal/config/section.go

@@ -1,12 +1,16 @@
 package config
 
 import (
+	"encoding/json"
 	"errors"
 	"fmt"
 	"reflect"
+	"slices"
 	"strconv"
 	"strings"
 	"time"
+
+	"git.avsdev.uk/Go/job-search/internal/util"
 )
 
 // Configuration sections are plain structs described with struct tags:
@@ -22,6 +26,19 @@ import (
 //
 // time.Duration fields report reflect.String from TypeOf and accept either a
 // time.Duration or a duration string such as "90s" in SetValue.
+// util.ByteSize fields work the same way, with size strings such as "32KiB".
+//
+// []string fields report reflect.Slice and accept either a []string or a
+// comma-separated string such as "a, b" in SetValue (and so in default tags).
+// An empty list is stored as nil.
+//
+// A list of sections, []S where S is a section struct (e.g. a list of extra
+// headers), is a single key reporting reflect.Slice. It is given out as a
+// []map[string]any, one map per item as Map gives it, and SetValue accepts
+// a []S, a list of maps (as Viper reads a YAML or JSON list of objects) or a
+// JSON array string. Each item starts from its defaults; its keys are set as
+// SetValue sets them, so values may be strings to parse. If the item type has
+// a validator, every field of each item is validated, including ones not set.
 //
 // A section can check values before they are stored by implementing
 // validator on its pointer type.
@@ -31,16 +48,18 @@ const (
 )
 
 var (
-	errUnknownKey = errors.New("unknown configuration key")
-	durationType  = reflect.TypeOf(time.Duration(0))
-	timeType      = reflect.TypeOf(time.Time{})
+	errUnknownKey   = errors.New("unknown configuration key")
+	durationType    = reflect.TypeOf(time.Duration(0))
+	byteSizeType    = reflect.TypeOf(util.ByteSize(0))
+	timeType        = reflect.TypeOf(time.Time{})
+	stringSliceType = reflect.TypeOf([]string{})
 )
 
 // Section is the method set of every configuration section, including
 // JobSearchConfig itself, implemented on the section's pointer type. Keys are
 // dotted paths relative to the section, e.g. "sleeper.interval" in
-// JobSearchConfigSources. Durations are given out as strings such as "10m0s",
-// matching TypeOf's reflect.String for them. TestSections checks every
+// JSCSources. Durations and byte sizes are given out as strings such as
+// "10m0s" and "32KiB", matching TypeOf's reflect.String for them. TestSections checks every
 // section, found by walking JobSearchConfig, implements it.
 type Section interface {
 	// Map returns the section as a map of keys to values, with nested
@@ -48,8 +67,9 @@ type Section interface {
 	Map() map[string]any
 	// Flatten returns the section as a flat map of dotted keys, each
 	// prefixed with the section's own key (its JSCK* constant), so a
-	// section's keys match how its parent flattens it. JobSearchConfig and
-	// the embedded JobSearchConfigSource have no prefix.
+	// section's keys match how its parent flattens it. JobSearchConfig, the
+	// embedded JSCSourcesSource and types used under several keys, such as
+	// JSCRateLimitRule, have no prefix.
 	Flatten() map[string]any
 	// TypeOf returns the kind of the value at key, which is how it is read
 	// from Viper, or an error for an unknown key.
@@ -80,6 +100,13 @@ type field struct {
 	hasDefault bool
 }
 
+// isSectionList reports whether t is a list of sections, []S for a section
+// struct S.
+func isSectionList(t reflect.Type) bool {
+	return t.Kind() == reflect.Slice && t.Elem().Kind() == reflect.Struct &&
+		t.Elem() != timeType
+}
+
 // fieldsOf lists the configuration fields of a section type in declaration
 // order, with the fields of embedded structs promoted into it.
 func fieldsOf(t reflect.Type) []field {
@@ -201,19 +228,48 @@ func flattenInto(m map[string]any, prefix string, section reflect.Value) {
 }
 
 // exportValue returns a field's value as Map, Flatten and ValueOf give it
-// out: durations become strings such as "1m30s", so values agree with kindOf
-// and can be passed back to SetValue; everything else is unchanged.
+// out: durations and byte sizes become strings such as "1m30s" and "32KiB",
+// so values agree with kindOf and can be passed back to SetValue; []string is
+// copied; a list of sections becomes a []map[string]any (nil if empty);
+// everything else is unchanged.
 func exportValue(fv reflect.Value) any {
-	if fv.Type() == durationType {
+	switch fv.Type() {
+	case durationType:
 		return time.Duration(fv.Int()).String()
+	case byteSizeType:
+		return util.ByteSize(fv.Int()).String()
 	}
-	return fv.Interface()
+	if isSectionList(fv.Type()) {
+		var items []map[string]any
+		for i := range fv.Len() {
+			items = append(items, mapOfValue(fv.Index(i)))
+		}
+		return items
+	}
+	v := fv.Interface()
+	if s, ok := v.([]string); ok {
+		// A copy, so changing it doesn't change the configuration.
+		return slices.Clone(s)
+	}
+	return v
+}
+
+// splitList splits a comma-separated list, trimming spaces around each item
+// and dropping empty ones, so "" is an empty list.
+func splitList(s string) []string {
+	var items []string
+	for item := range strings.SplitSeq(s, ",") {
+		if item = strings.TrimSpace(item); item != "" {
+			items = append(items, item)
+		}
+	}
+	return items
 }
 
 // kindOf is the kind reported by TypeOf, i.e. how the value is read from
-// Viper. Durations are read as strings and parsed by SetValue.
+// Viper. Durations and byte sizes are read as strings and parsed by SetValue.
 func kindOf(t reflect.Type) reflect.Kind {
-	if t == durationType {
+	if t == durationType || t == byteSizeType {
 		return reflect.String
 	}
 	return t.Kind()
@@ -265,11 +321,19 @@ func setField(section reflect.Value, f field, value any) error {
 }
 
 // convert turns value into a value of type t. Values already of the right
-// type are used as-is; strings are parsed.
+// type are used as-is, except that an empty []string becomes nil, as an empty
+// comma-separated list does (see splitList); strings are parsed. Lists of
+// sections are converted by convertSectionList.
 func convert(value any, t reflect.Type) (reflect.Value, error) {
+	if isSectionList(t) {
+		return convertSectionList(value, t)
+	}
 	if value != nil {
 		rv := reflect.ValueOf(value)
 		if rv.Type().AssignableTo(t) {
+			if t == stringSliceType && rv.Len() == 0 {
+				return reflect.Zero(t), nil
+			}
 			return rv, nil
 		}
 		if s, ok := value.(string); ok {
@@ -279,17 +343,123 @@ func convert(value any, t reflect.Type) (reflect.Value, error) {
 	return reflect.Value{}, fmt.Errorf("unsupported type %T", value)
 }
 
+// convertSectionList turns value into a list of sections of type t: a value
+// of type t, a list of maps of keys to values, or a JSON array of objects.
+// nil and empty lists become a nil list.
+func convertSectionList(value any, t reflect.Type) (reflect.Value, error) {
+	if s, ok := value.(string); ok {
+		var items []map[string]any
+		if strings.TrimSpace(s) != "" {
+			if err := json.Unmarshal([]byte(s), &items); err != nil {
+				return reflect.Value{}, fmt.Errorf("unable to convert to a list of %s: %w", t.Elem(), err)
+			}
+		}
+		value = items
+	}
+	if value == nil {
+		return reflect.Zero(t), nil
+	}
+	rv := reflect.ValueOf(value)
+	if rv.Kind() != reflect.Slice {
+		return reflect.Value{}, fmt.Errorf("unsupported type %T", value)
+	}
+	if rv.Len() == 0 {
+		return reflect.Zero(t), nil
+	}
+	if rv.Type().AssignableTo(t) {
+		for i := range rv.Len() {
+			if err := validateSection(rv.Index(i)); err != nil {
+				return reflect.Value{}, fmt.Errorf("item %d: %w", i, err)
+			}
+		}
+		return rv, nil
+	}
+	list := reflect.MakeSlice(t, 0, rv.Len())
+	for i := range rv.Len() {
+		item, err := sectionFromMap(rv.Index(i).Interface(), t.Elem())
+		if err != nil {
+			return reflect.Value{}, fmt.Errorf("item %d: %w", i, err)
+		}
+		list = reflect.Append(list, item)
+	}
+	return list, nil
+}
+
+// sectionFromMap builds a section of type t from a map of its keys to values,
+// starting from its defaults. Values that are neither strings nor of the
+// field's type (e.g. numbers read from YAML) are formatted and parsed. The
+// finished section is validated as a whole (see validateSection), so a key
+// left out of the map can't skip its validation.
+func sectionFromMap(value any, t reflect.Type) (reflect.Value, error) {
+	item, ok := value.(map[string]any)
+	if !ok {
+		return reflect.Value{}, fmt.Errorf("unsupported type %T", value)
+	}
+	section := reflect.New(t).Elem()
+	if err := applyDefaultsTo("", section); err != nil {
+		return reflect.Value{}, err
+	}
+	for k, v := range item {
+		s, f, err := lookup(section, k)
+		if err != nil {
+			return reflect.Value{}, fmt.Errorf("%w %q", err, k)
+		}
+		if _, isString := v.(string); v != nil && !isString &&
+			!reflect.TypeOf(v).AssignableTo(f.typ) {
+			v = fmt.Sprint(v)
+		}
+		if err := setField(s, f, v); err != nil {
+			return reflect.Value{}, fmt.Errorf("%s: %w", k, err)
+		}
+	}
+	if err := validateSection(section); err != nil {
+		return reflect.Value{}, err
+	}
+	return section, nil
+}
+
+// validateSection runs the section's validator, if it has one, on every value
+// field directly in it, so values that were never set (defaults or zero
+// values) are checked too.
+func validateSection(section reflect.Value) error {
+	v, ok := reflect.New(section.Type()).Interface().(validator)
+	if !ok {
+		return nil
+	}
+	for _, f := range fieldsOf(section.Type()) {
+		if f.section {
+			continue
+		}
+		if err := v.validate(f.key, section.FieldByIndex(f.index).Interface()); err != nil {
+			return fmt.Errorf("%s: %w", f.key, err)
+		}
+	}
+	return nil
+}
+
 // parseString parses s as a value of type t. It supports strings, bools,
-// signed and unsigned integers, floats and time.Duration.
+// signed and unsigned integers, floats, time.Duration, ByteSize and []string
+// (from a comma-separated list, see splitList).
 func parseString(s string, t reflect.Type) (reflect.Value, error) {
 	v := reflect.New(t).Elem()
-	if t == durationType {
+	switch t {
+	case durationType:
 		d, err := time.ParseDuration(s)
 		if err != nil {
 			return reflect.Value{}, fmt.Errorf("unable to convert to duration: %w", err)
 		}
 		v.SetInt(int64(d))
 		return v, nil
+	case byteSizeType:
+		b, err := util.ParseByteSize(s)
+		if err != nil {
+			return reflect.Value{}, fmt.Errorf("unable to convert to byte size: %w", err)
+		}
+		v.SetInt(int64(b))
+		return v, nil
+	case stringSliceType:
+		v.Set(reflect.ValueOf(splitList(s)))
+		return v, nil
 	}
 
 	var err error

internal/config/section_test.go

@@ -6,6 +6,8 @@ import (
 	"strings"
 	"testing"
 	"time"
+
+	"git.avsdev.uk/Go/job-search/internal/util"
 )
 
 // Test sections exercising every field shape section.go supports.
@@ -13,6 +15,13 @@ type testInner struct {
 	Name string `cfg:"name" default:"inner"`
 }
 
+// testItem is the item type of a list of sections.
+type testItem struct {
+	Name string        `cfg:"name" default:"item"`
+	N    int           `cfg:"n"`
+	Wait time.Duration `cfg:"wait"`
+}
+
 type testEmbedded struct {
 	On bool `cfg:"on" default:"true"`
 }
@@ -27,8 +36,10 @@ type testSection struct {
 	Float    float64       `cfg:"float"      default:"1.5"`
 	Float32  float32       `cfg:"float32"`
 	Dur      time.Duration `cfg:"dur"        default:"90s"`
+	Size     util.ByteSize `cfg:"size"       default:"32KiB"`
 	When     time.Time     `cfg:"when"`
 	Slice    []string      `cfg:"slice"`
+	Items    []testItem    `cfg:"items"`
 	Inner    testInner     `cfg:"inner"`
 	NoTag    string
 	Skipped  string `cfg:"-"`
@@ -52,7 +63,7 @@ func (c *testValidated) validate(key string, value any) error {
 func TestKeysOf(t *testing.T) {
 	got := keysOf(&testSection{})
 	want := []string{"on", "str", "int", "int8", "uint", "uint16", "float",
-		"float32", "dur", "when", "slice", "inner"}
+		"float32", "dur", "size", "when", "slice", "items", "inner"}
 	if !reflect.DeepEqual(got, want) {
 		t.Errorf("keysOf = %v; want %v", got, want)
 	}
@@ -70,6 +81,9 @@ func TestFieldsOfSkipsUntagged(t *testing.T) {
 		if f.key == "inner" && !f.section {
 			t.Error("struct field not treated as a section")
 		}
+		if f.key == "items" && f.section {
+			t.Error("list of sections treated as a section")
+		}
 	}
 }
 
@@ -137,6 +151,7 @@ func TestApplyDefaults(t *testing.T) {
 		Uint:         7,
 		Float:        1.5,
 		Dur:          90 * time.Second,
+		Size:         32 << 10,
 		Inner:        testInner{Name: "inner"},
 	}
 	if !reflect.DeepEqual(s, want) {
@@ -174,9 +189,11 @@ func TestTypeOf(t *testing.T) {
 		{"uint", reflect.Uint},
 		{"float", reflect.Float64},
 		{"on", reflect.Bool},
-		{"dur", reflect.String}, // durations are read as strings
+		{"dur", reflect.String},  // durations are read as strings
+		{"size", reflect.String}, // byte sizes are read as strings
 		{"when", reflect.Struct},
 		{"slice", reflect.Slice},
+		{"items", reflect.Slice},
 		{"inner.name", reflect.String},
 	}
 	for _, tt := range tests {
@@ -227,12 +244,38 @@ func TestSetValue(t *testing.T) {
 		{"float", "2.25", 2.25},
 		{"float32", "0.5", float32(0.5)},
 		{"dur", "1h30m", "1h30m0s"}, // durations are given out as strings
+		{"size", "1 mb", "1MB"},     // byte sizes are given out as strings
+		{"size", "1.5KiB", "1536B"},
+		{"size", "4096", "4KiB"},
+		{"slice", " a, b ,, c ", []string{"a", "b", "c"}},
+		{"slice", "", []string(nil)},
 		{"inner.name", "nested", "nested"},
 		// values of the field's type are used as-is
 		{"int", 5, 5},
 		{"on", true, true},
 		{"dur", 3 * time.Second, "3s"},
+		{"size", util.ByteSize(2048), "2KiB"},
 		{"slice", []string{"a", "b"}, []string{"a", "b"}},
+		{"slice", []string{}, []string(nil)}, // empty lists are stored as nil
+		// lists of sections are given out as lists of maps; items start
+		// from their defaults and their values are parsed
+		{"items", `[{"n": "1", "wait": "2s"}, {"name": "b", "n": 2}]`, []map[string]any{
+			{"name": "item", "n": 1, "wait": "2s"},
+			{"name": "b", "n": 2, "wait": "0s"},
+		}},
+		{"items", []any{map[string]any{"name": "a", "n": 3}}, []map[string]any{
+			{"name": "a", "n": 3, "wait": "0s"},
+		}},
+		{"items", []map[string]any{{"name": "a"}}, []map[string]any{
+			{"name": "a", "n": 0, "wait": "0s"},
+		}},
+		{"items", []testItem{{Name: "a", N: 4}}, []map[string]any{
+			{"name": "a", "n": 4, "wait": "0s"},
+		}},
+		{"items", "", []map[string]any(nil)},
+		{"items", "[]", []map[string]any(nil)},
+		{"items", []testItem{}, []map[string]any(nil)},
+		{"items", nil, []map[string]any(nil)},
 		{"when", time.Unix(0, 0).UTC(), time.Unix(0, 0).UTC()},
 	}
 	for _, tt := range tests {
@@ -261,7 +304,14 @@ func TestSetValueErrors(t *testing.T) {
 		{"float", "x", "unable to convert"},
 		{"on", "maybe", "unable to convert"},
 		{"dur", "soon", "unable to convert to duration"},
-		{"slice", "a,b", "unsupported type string"},
+		{"items", "not json", "unable to convert to a list of config.testItem"},
+		{"items", `[{"bogus": 1}]`, `item 0: unknown configuration key "bogus"`},
+		{"items", `[{"n": "x"}]`, "item 0: n: unable to convert"},
+		{"items", []any{"a"}, "item 0: unsupported type string"},
+		{"items", 5, "unsupported type int"},
+		{"size", "lots", "unable to convert to byte size"},
+		{"size", "-1KB", "unable to convert to byte size"},
+		{"size", "16EiB", "too large"},
 		{"when", "2024-01-01", "unsupported type string"},
 		{"int", 1.5, "unsupported type float64"},
 		{"int", int64(1), "unsupported type int64"},
@@ -281,6 +331,20 @@ func TestSetValueErrors(t *testing.T) {
 	}
 }
 
+// A []string is given out as a copy, so changing it leaves the section as it
+// was.
+func TestValueOfSliceIsACopy(t *testing.T) {
+	s := testSection{Slice: []string{"a", "b"}}
+	got, err := valueOf(s, "slice")
+	if err != nil {
+		t.Fatal(err)
+	}
+	got.([]string)[0] = "changed"
+	if s.Slice[0] != "a" {
+		t.Errorf("changing valueOf's slice changed the section: %v", s.Slice)
+	}
+}
+
 func TestSetValueValidator(t *testing.T) {
 	var s testValidated
 	if err := setValue(&s, "value", "good"); err != nil || s.Value != "good" {