bwilliams
b0f93fd437
A section tagged init:"hidden" isn't written to the file config init generates; fts_mock and sleeper are. config init --all writes them too. They still load, and the config commands read and set them as before.
604 lines
19 KiB
Go
604 lines
19 KiB
Go
package config
|
|
|
|
import (
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"reflect"
|
|
"slices"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
|
|
"git.avsdev.uk/Go/job-search/internal/util"
|
|
)
|
|
|
|
// Configuration sections are plain structs described with struct tags:
|
|
//
|
|
// cfg:"key" the field's configuration key. Fields without it are
|
|
// not part of the configuration.
|
|
// default:"value" the field's default, parsed the same way as a string
|
|
// passed to SetValue (e.g. "10m", "true", "42").
|
|
//
|
|
// A tagged field whose type is a struct (other than time.Time) is a nested
|
|
// section, addressed as "section.key". An embedded struct has its tagged
|
|
// fields promoted into the enclosing section, just as Go promotes fields.
|
|
//
|
|
// time.Duration fields report reflect.String from TypeOf and accept either a
|
|
// time.Duration or a duration string such as "90s" in SetValue.
|
|
// util.ByteSize fields work the same way, with size strings such as "32KiB"
|
|
// (or "-1" for no limit).
|
|
//
|
|
// []string fields report reflect.Slice and accept either a []string or a
|
|
// comma-separated string such as "a, b" in SetValue (and so in default tags).
|
|
// An empty list is stored as nil.
|
|
//
|
|
// A list of sections, []S where S is a section struct (e.g. a list of extra
|
|
// headers), is a single key reporting reflect.Slice. It is given out as a
|
|
// []map[string]any, one map per item as Map gives it, and SetValue accepts
|
|
// a []S, a list of maps (as Viper reads a YAML or JSON list of objects) or a
|
|
// JSON array string. Each item starts from its defaults; its keys are set as
|
|
// SetValue sets them, so values may be strings to parse. If the item type has
|
|
// a validator, every field of each item is validated, including ones not set.
|
|
//
|
|
// A section can check values before they are stored by implementing
|
|
// validator on its pointer type.
|
|
|
|
// The struct tags section.go reads.
|
|
const (
|
|
// keyTag holds a field's configuration key.
|
|
keyTag = "cfg"
|
|
// defaultTag holds a field's default value.
|
|
defaultTag = "default"
|
|
// initTag set to initHidden on a section field leaves the section out of
|
|
// the file `config init` writes (see InitDefaults), as for test sources.
|
|
initTag = "init"
|
|
initHidden = "hidden"
|
|
)
|
|
|
|
// The unknown key error and the field types section.go handles specially.
|
|
var (
|
|
// errUnknownKey is returned for a key that names no field.
|
|
errUnknownKey = errors.New("config: unknown configuration key")
|
|
// durationType is time.Duration, read and given out as a string.
|
|
durationType = reflect.TypeOf(time.Duration(0))
|
|
// byteSizeType is util.ByteSize, read and given out as a string.
|
|
byteSizeType = reflect.TypeOf(util.ByteSize(0))
|
|
// timeType is time.Time, a struct that is a value, not a section.
|
|
timeType = reflect.TypeOf(time.Time{})
|
|
// stringSliceType is []string, set from a comma-separated list.
|
|
stringSliceType = reflect.TypeOf([]string{})
|
|
)
|
|
|
|
// Section is the method set of every configuration section, including
|
|
// JobSearchConfig itself, implemented on the section's pointer type. Keys are
|
|
// dotted paths relative to the section, e.g. "sleeper.interval" in JSCSources.
|
|
// Durations and byte sizes are given out as strings such as "10m0s" and
|
|
// "32KiB", matching TypeOf's reflect.String for them. TestSections checks every
|
|
// section, found by walking JobSearchConfig, implements it.
|
|
type Section interface {
|
|
// Map returns the section as a map of keys to values, with nested
|
|
// sections as nested maps.
|
|
Map() map[string]any
|
|
// Flatten returns the section as a flat map of dotted keys, each
|
|
// prefixed with the section's own key (its JSCK* constant), so a
|
|
// section's keys match how its parent flattens it. JobSearchConfig, the
|
|
// embedded JSCSourcesSource and types used under several keys, such as
|
|
// JSCRateLimitRule, have no prefix.
|
|
Flatten() map[string]any
|
|
// TypeOf returns the kind of the value at key, which is how it is read
|
|
// from Viper, or an error for an unknown key.
|
|
TypeOf(key string) (reflect.Kind, error)
|
|
// ValueOf returns the value at key, or an error for an unknown key.
|
|
ValueOf(key string) (any, error)
|
|
// SetValue stores value at key. value may be of the field's type or a
|
|
// string to parse (e.g. "10m", "true", "42"). The section's validation,
|
|
// if any, runs first; on error the field is unchanged.
|
|
SetValue(key string, value any) error
|
|
}
|
|
|
|
// validator is implemented by sections that need to reject some values.
|
|
// key is the field's own key within the section; value has already been
|
|
// converted to the field's type.
|
|
type validator interface {
|
|
validate(key string, value any) error
|
|
}
|
|
|
|
// field is a section struct field that is part of the configuration (see
|
|
// fieldsOf).
|
|
type field struct {
|
|
key string
|
|
index []int // path for reflect.Value.FieldByIndex
|
|
fieldType reflect.Type
|
|
section bool
|
|
defaultValue string
|
|
hasDefault bool
|
|
// hiddenFromInit is set by initTag (see InitDefaults).
|
|
hiddenFromInit bool
|
|
}
|
|
|
|
// isSectionList reports whether listType is a list of sections, []S for a
|
|
// section struct S.
|
|
func isSectionList(listType reflect.Type) bool {
|
|
return listType.Kind() == reflect.Slice &&
|
|
listType.Elem().Kind() == reflect.Struct &&
|
|
listType.Elem() != timeType
|
|
}
|
|
|
|
// fieldsOf lists the configuration fields of a section type in declaration
|
|
// order, with the fields of embedded structs promoted into it.
|
|
func fieldsOf(structType reflect.Type) []field {
|
|
var fields []field
|
|
for i := 0; i < structType.NumField(); i++ {
|
|
structField := structType.Field(i)
|
|
|
|
if structField.Anonymous && structField.Type.Kind() == reflect.Struct {
|
|
for _, embeddedField := range fieldsOf(structField.Type) {
|
|
embeddedField.index = append([]int{i}, embeddedField.index...)
|
|
fields = append(fields, embeddedField)
|
|
}
|
|
continue
|
|
}
|
|
|
|
key, ok := structField.Tag.Lookup(keyTag)
|
|
if !ok || key == "" || key == "-" || !structField.IsExported() {
|
|
continue
|
|
}
|
|
|
|
defaultValue, hasDefault := structField.Tag.Lookup(defaultTag)
|
|
fields = append(fields, field{
|
|
key: key,
|
|
index: []int{i},
|
|
fieldType: structField.Type,
|
|
section: structField.Type.Kind() == reflect.Struct &&
|
|
structField.Type != timeType,
|
|
defaultValue: defaultValue,
|
|
hasDefault: hasDefault,
|
|
hiddenFromInit: structField.Tag.Get(initTag) == initHidden,
|
|
})
|
|
}
|
|
|
|
return fields
|
|
}
|
|
|
|
// structValue dereferences value down to the section struct it holds.
|
|
func structValue(value any) reflect.Value {
|
|
reflected := reflect.ValueOf(value)
|
|
for reflected.Kind() == reflect.Pointer {
|
|
reflected = reflected.Elem()
|
|
}
|
|
if reflected.Kind() != reflect.Struct {
|
|
panic(fmt.Sprintf("config: %T is not a configuration section", value))
|
|
}
|
|
return reflected
|
|
}
|
|
|
|
// lookup resolves a dotted key to the leaf field and the section holding it.
|
|
func lookup(section reflect.Value, key string) (reflect.Value, field, error) {
|
|
head, rest, nested := strings.Cut(key, ".")
|
|
for _, configField := range fieldsOf(section.Type()) {
|
|
if configField.key != head {
|
|
continue
|
|
}
|
|
switch {
|
|
case configField.section && nested:
|
|
return lookup(section.FieldByIndex(configField.index), rest)
|
|
case !configField.section && !nested:
|
|
return section, configField, nil
|
|
}
|
|
// Key matched but the shape didn't: a section with no sub-key, or a
|
|
// value with one.
|
|
return reflect.Value{}, field{}, errUnknownKey
|
|
}
|
|
return reflect.Value{}, field{}, errUnknownKey
|
|
}
|
|
|
|
func joinKey(prefix, key string) string {
|
|
if prefix == "" {
|
|
return key
|
|
}
|
|
return prefix + "." + key
|
|
}
|
|
|
|
// keysOf returns the keys directly in a section.
|
|
func keysOf(value any) []string {
|
|
fields := fieldsOf(structValue(value).Type())
|
|
keys := make([]string, 0, len(fields))
|
|
for _, configField := range fields {
|
|
keys = append(keys, configField.key)
|
|
}
|
|
return keys
|
|
}
|
|
|
|
// mapOf converts a section to a map, with nested sections as nested maps.
|
|
func mapOf(value any) map[string]any {
|
|
return mapOfValue(structValue(value))
|
|
}
|
|
|
|
// mapOfValue is mapOf for a section struct value.
|
|
func mapOfValue(section reflect.Value) map[string]any {
|
|
values := map[string]any{}
|
|
for _, configField := range fieldsOf(section.Type()) {
|
|
fieldValue := section.FieldByIndex(configField.index)
|
|
if configField.section {
|
|
values[configField.key] = mapOfValue(fieldValue)
|
|
} else {
|
|
values[configField.key] = exportValue(fieldValue)
|
|
}
|
|
}
|
|
return values
|
|
}
|
|
|
|
// flattenOf converts a section to a flat map of dotted keys, each prefixed
|
|
// with prefix (if non-empty).
|
|
func flattenOf(prefix string, value any) map[string]any {
|
|
flat := map[string]any{}
|
|
flattenInto(flat, prefix, structValue(value))
|
|
return flat
|
|
}
|
|
|
|
// flattenInto adds section's keys to m, dotted and prefixed with prefix (if
|
|
// non-empty), descending into nested sections.
|
|
func flattenInto(flat map[string]any, prefix string, section reflect.Value) {
|
|
for _, configField := range fieldsOf(section.Type()) {
|
|
fieldValue := section.FieldByIndex(configField.index)
|
|
key := joinKey(prefix, configField.key)
|
|
if configField.section {
|
|
flattenInto(flat, key, fieldValue)
|
|
} else {
|
|
flat[key] = exportValue(fieldValue)
|
|
}
|
|
}
|
|
}
|
|
|
|
// hiddenFromInitKeys returns the dotted keys of the sections under section
|
|
// (prefixed with prefix, if non-empty) whose field has initTag set to
|
|
// initHidden, descending into nested sections.
|
|
func hiddenFromInitKeys(prefix string, sectionType reflect.Type) []string {
|
|
var hidden []string
|
|
for _, configField := range fieldsOf(sectionType) {
|
|
if !configField.section {
|
|
continue
|
|
}
|
|
key := joinKey(prefix, configField.key)
|
|
if configField.hiddenFromInit {
|
|
hidden = append(hidden, key)
|
|
continue
|
|
}
|
|
hidden = append(hidden, hiddenFromInitKeys(key, configField.fieldType)...)
|
|
}
|
|
return hidden
|
|
}
|
|
|
|
// exportValue returns a field's value as Map, Flatten and ValueOf give it
|
|
// out: durations and byte sizes become strings such as "1m30s" and "32KiB",
|
|
// so values agree with kindOf and can be passed back to SetValue; []string is
|
|
// copied; a list of sections becomes a []map[string]any (nil if empty);
|
|
// everything else is unchanged.
|
|
func exportValue(fieldValue reflect.Value) any {
|
|
switch fieldValue.Type() {
|
|
case durationType:
|
|
return time.Duration(fieldValue.Int()).String()
|
|
case byteSizeType:
|
|
return util.ByteSize(fieldValue.Int()).String()
|
|
}
|
|
|
|
if isSectionList(fieldValue.Type()) {
|
|
var items []map[string]any
|
|
for i := range fieldValue.Len() {
|
|
items = append(items, mapOfValue(fieldValue.Index(i)))
|
|
}
|
|
return items
|
|
}
|
|
|
|
value := fieldValue.Interface()
|
|
if list, ok := value.([]string); ok {
|
|
// A copy, so changing it doesn't change the configuration.
|
|
return slices.Clone(list)
|
|
}
|
|
|
|
return value
|
|
}
|
|
|
|
// splitList splits a comma-separated list, trimming spaces around each item
|
|
// and dropping empty ones, so "" is an empty list.
|
|
func splitList(list string) []string {
|
|
var items []string
|
|
for item := range strings.SplitSeq(list, ",") {
|
|
if item = strings.TrimSpace(item); item != "" {
|
|
items = append(items, item)
|
|
}
|
|
}
|
|
return items
|
|
}
|
|
|
|
// kindOf is the kind reported by TypeOf, i.e. how the value is read from
|
|
// Viper. Durations and byte sizes are read as strings and parsed by SetValue.
|
|
func kindOf(fieldType reflect.Type) reflect.Kind {
|
|
if fieldType == durationType || fieldType == byteSizeType {
|
|
return reflect.String
|
|
}
|
|
return fieldType.Kind()
|
|
}
|
|
|
|
// typeOf returns the kind of the dotted key in the section value (see
|
|
// kindOf), or errUnknownKey.
|
|
func typeOf(value any, key string) (reflect.Kind, error) {
|
|
_, configField, err := lookup(structValue(value), key)
|
|
if err != nil {
|
|
return reflect.Invalid, err
|
|
}
|
|
return kindOf(configField.fieldType), nil
|
|
}
|
|
|
|
// valueOf returns the value of the dotted key in the section value, or
|
|
// errUnknownKey.
|
|
func valueOf(value any, key string) (any, error) {
|
|
section, configField, err := lookup(structValue(value), key)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return exportValue(section.FieldByIndex(configField.index)), nil
|
|
}
|
|
|
|
// setValue sets a dotted key on the section that ptr points to.
|
|
func setValue(ptr any, key string, value any) error {
|
|
section, configField, err := lookup(structValue(ptr), key)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return setField(section, configField, value)
|
|
}
|
|
|
|
// setField converts value to the field's type and, if the section has a
|
|
// validator, checks it, before setting the field. On error the field is left
|
|
// unchanged.
|
|
func setField(section reflect.Value, configField field, value any) error {
|
|
newValue, err := convert(value, configField.fieldType)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
if checker, ok := section.Addr().Interface().(validator); ok {
|
|
err := checker.validate(configField.key, newValue.Interface())
|
|
if err != nil {
|
|
return err
|
|
}
|
|
}
|
|
section.FieldByIndex(configField.index).Set(newValue)
|
|
|
|
return nil
|
|
}
|
|
|
|
// convert turns value into a value of type targetType. Values already of the
|
|
// right type are used as-is, except that an empty []string becomes nil, as an
|
|
// empty comma-separated list does (see splitList); strings are parsed. Lists of
|
|
// sections are converted by convertSectionList.
|
|
func convert(value any, targetType reflect.Type) (reflect.Value, error) {
|
|
if isSectionList(targetType) {
|
|
return convertSectionList(value, targetType)
|
|
}
|
|
if value != nil {
|
|
reflected := reflect.ValueOf(value)
|
|
if reflected.Type().AssignableTo(targetType) {
|
|
if targetType == stringSliceType && reflected.Len() == 0 {
|
|
return reflect.Zero(targetType), nil
|
|
}
|
|
return reflected, nil
|
|
}
|
|
if text, ok := value.(string); ok {
|
|
return parseString(text, targetType)
|
|
}
|
|
}
|
|
return reflect.Value{}, fmt.Errorf("config: unsupported type %T", value)
|
|
}
|
|
|
|
// convertSectionList turns value into a list of sections of type listType: a
|
|
// value of type listType, a list of maps of keys to values, or a JSON array of
|
|
// objects. nil and empty lists become a nil list.
|
|
func convertSectionList(
|
|
value any, listType reflect.Type,
|
|
) (reflect.Value, error) {
|
|
if text, ok := value.(string); ok {
|
|
var items []map[string]any
|
|
if strings.TrimSpace(text) != "" {
|
|
if err := json.Unmarshal([]byte(text), &items); err != nil {
|
|
return reflect.Value{}, fmt.Errorf(
|
|
"unable to convert to a list of %s: %w", listType.Elem(), err,
|
|
)
|
|
}
|
|
}
|
|
value = items
|
|
}
|
|
|
|
if value == nil {
|
|
return reflect.Zero(listType), nil
|
|
}
|
|
reflected := reflect.ValueOf(value)
|
|
if reflected.Kind() != reflect.Slice {
|
|
return reflect.Value{}, fmt.Errorf("config: unsupported type %T", value)
|
|
}
|
|
if reflected.Len() == 0 {
|
|
return reflect.Zero(listType), nil
|
|
}
|
|
|
|
if reflected.Type().AssignableTo(listType) {
|
|
for i := range reflected.Len() {
|
|
if err := validateSection(reflected.Index(i)); err != nil {
|
|
return reflect.Value{}, fmt.Errorf("item %d: %w", i, err)
|
|
}
|
|
}
|
|
return reflected, nil
|
|
}
|
|
|
|
list := reflect.MakeSlice(listType, 0, reflected.Len())
|
|
for i := range reflected.Len() {
|
|
item, err := sectionFromMap(reflected.Index(i).Interface(), listType.Elem())
|
|
if err != nil {
|
|
return reflect.Value{}, fmt.Errorf("item %d: %w", i, err)
|
|
}
|
|
list = reflect.Append(list, item)
|
|
}
|
|
|
|
return list, nil
|
|
}
|
|
|
|
// sectionFromMap builds a section of type structType from value, a map of its
|
|
// keys to values, starting from its defaults. Values that are neither strings
|
|
// nor of the field's type (e.g. numbers read from YAML) are formatted and
|
|
// parsed. The finished section is validated as a whole (see
|
|
// validateSection), so a key left out of the map can't skip its validation.
|
|
func sectionFromMap(value any, structType reflect.Type) (reflect.Value, error) {
|
|
item, ok := value.(map[string]any)
|
|
if !ok {
|
|
return reflect.Value{}, fmt.Errorf("config: unsupported type %T", value)
|
|
}
|
|
|
|
section := reflect.New(structType).Elem()
|
|
if err := applyDefaultsTo("", section); err != nil {
|
|
return reflect.Value{}, err
|
|
}
|
|
|
|
for itemKey, itemValue := range item {
|
|
holder, configField, err := lookup(section, itemKey)
|
|
if err != nil {
|
|
return reflect.Value{}, fmt.Errorf("%w %q", err, itemKey)
|
|
}
|
|
if _, isString := itemValue.(string); itemValue != nil && !isString &&
|
|
!reflect.TypeOf(itemValue).AssignableTo(configField.fieldType) {
|
|
itemValue = fmt.Sprint(itemValue)
|
|
}
|
|
if err := setField(holder, configField, itemValue); err != nil {
|
|
return reflect.Value{}, fmt.Errorf("%s: %w", itemKey, err)
|
|
}
|
|
}
|
|
|
|
if err := validateSection(section); err != nil {
|
|
return reflect.Value{}, err
|
|
}
|
|
|
|
return section, nil
|
|
}
|
|
|
|
// validateSection runs the section's validator, if it has one, on every value
|
|
// field directly in it, so values that were never set (defaults or zero
|
|
// values) are checked too.
|
|
func validateSection(section reflect.Value) error {
|
|
checker, ok := reflect.New(section.Type()).Interface().(validator)
|
|
if !ok {
|
|
return nil
|
|
}
|
|
|
|
for _, configField := range fieldsOf(section.Type()) {
|
|
if configField.section {
|
|
continue
|
|
}
|
|
value := section.FieldByIndex(configField.index).Interface()
|
|
if err := checker.validate(configField.key, value); err != nil {
|
|
return fmt.Errorf("%s: %w", configField.key, err)
|
|
}
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
// parseString parses text as a value of type targetType. It supports strings,
|
|
// bools, signed and unsigned integers, floats, time.Duration, ByteSize and
|
|
// []string (from a comma-separated list, see splitList).
|
|
func parseString(text string, targetType reflect.Type) (reflect.Value, error) {
|
|
parsed := reflect.New(targetType).Elem()
|
|
switch targetType {
|
|
case durationType:
|
|
duration, err := time.ParseDuration(text)
|
|
if err != nil {
|
|
return reflect.Value{}, fmt.Errorf(
|
|
"unable to convert to duration: %w", err,
|
|
)
|
|
}
|
|
parsed.SetInt(int64(duration))
|
|
return parsed, nil
|
|
case byteSizeType:
|
|
size, err := util.ParseByteSize(text)
|
|
if err != nil {
|
|
return reflect.Value{}, fmt.Errorf(
|
|
"unable to convert to byte size: %w", err,
|
|
)
|
|
}
|
|
parsed.SetInt(int64(size))
|
|
return parsed, nil
|
|
case stringSliceType:
|
|
parsed.Set(reflect.ValueOf(splitList(text)))
|
|
return parsed, nil
|
|
}
|
|
|
|
var err error
|
|
switch targetType.Kind() {
|
|
case reflect.String:
|
|
parsed.SetString(text)
|
|
case reflect.Bool:
|
|
var flag bool
|
|
flag, err = strconv.ParseBool(text)
|
|
parsed.SetBool(flag)
|
|
case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
|
|
var number int64
|
|
number, err = strconv.ParseInt(text, 10, targetType.Bits())
|
|
parsed.SetInt(number)
|
|
case reflect.Uint,
|
|
reflect.Uint8,
|
|
reflect.Uint16,
|
|
reflect.Uint32,
|
|
reflect.Uint64:
|
|
var number uint64
|
|
number, err = strconv.ParseUint(text, 10, targetType.Bits())
|
|
parsed.SetUint(number)
|
|
case reflect.Float32, reflect.Float64:
|
|
var number float64
|
|
number, err = strconv.ParseFloat(text, targetType.Bits())
|
|
parsed.SetFloat(number)
|
|
default:
|
|
return reflect.Value{}, fmt.Errorf(
|
|
"config: unsupported type string for %s", targetType,
|
|
)
|
|
}
|
|
if err != nil {
|
|
return reflect.Value{}, fmt.Errorf(
|
|
"unable to convert %q to %s: %w", text, targetType, err,
|
|
)
|
|
}
|
|
|
|
return parsed, nil
|
|
}
|
|
|
|
// applyDefaults sets every field that has a default tag, recursing into
|
|
// nested sections. ptr must point to a section struct.
|
|
func applyDefaults(ptr any) error {
|
|
return applyDefaultsTo("", structValue(ptr))
|
|
}
|
|
|
|
// applyDefaultsTo sets every field in section that has a default tag,
|
|
// recursing into nested sections. prefix is the section's dotted key, used in
|
|
// error messages.
|
|
func applyDefaultsTo(prefix string, section reflect.Value) error {
|
|
for _, configField := range fieldsOf(section.Type()) {
|
|
key := joinKey(prefix, configField.key)
|
|
|
|
if configField.section {
|
|
nested := section.FieldByIndex(configField.index)
|
|
if err := applyDefaultsTo(key, nested); err != nil {
|
|
return err
|
|
}
|
|
continue
|
|
}
|
|
|
|
if !configField.hasDefault {
|
|
continue
|
|
}
|
|
|
|
err := setField(section, configField, configField.defaultValue)
|
|
if err != nil {
|
|
return fmt.Errorf("invalid default for %s: %w", key, err)
|
|
}
|
|
}
|
|
return nil
|
|
}
|