package config import ( "encoding/json" "errors" "fmt" "reflect" "slices" "strconv" "strings" "time" "git.avsdev.uk/Go/job-search/internal/util" ) // Configuration sections are plain structs described with struct tags: // // cfg:"key" the field's configuration key. Fields without it are // not part of the configuration. // default:"value" the field's default, parsed the same way as a string // passed to SetValue (e.g. "10m", "true", "42"). // // A tagged field whose type is a struct (other than time.Time) is a nested // section, addressed as "section.key". An embedded struct has its tagged // fields promoted into the enclosing section, just as Go promotes fields. // // time.Duration fields report reflect.String from TypeOf and accept either a // time.Duration or a duration string such as "90s" in SetValue. // util.ByteSize fields work the same way, with size strings such as "32KiB". // // []string fields report reflect.Slice and accept either a []string or a // comma-separated string such as "a, b" in SetValue (and so in default tags). // An empty list is stored as nil. // // A list of sections, []S where S is a section struct (e.g. a list of extra // headers), is a single key reporting reflect.Slice. It is given out as a // []map[string]any, one map per item as Map gives it, and SetValue accepts // a []S, a list of maps (as Viper reads a YAML or JSON list of objects) or a // JSON array string. Each item starts from its defaults; its keys are set as // SetValue sets them, so values may be strings to parse. If the item type has // a validator, every field of each item is validated, including ones not set. // // A section can check values before they are stored by implementing // validator on its pointer type. const ( keyTag = "cfg" defaultTag = "default" ) var ( errUnknownKey = errors.New("unknown configuration key") durationType = reflect.TypeOf(time.Duration(0)) byteSizeType = reflect.TypeOf(util.ByteSize(0)) timeType = reflect.TypeOf(time.Time{}) stringSliceType = reflect.TypeOf([]string{}) ) // Section is the method set of every configuration section, including // JobSearchConfig itself, implemented on the section's pointer type. Keys are // dotted paths relative to the section, e.g. "sleeper.interval" in // JSCSources. Durations and byte sizes are given out as strings such as // "10m0s" and "32KiB", matching TypeOf's reflect.String for them. TestSections checks every // section, found by walking JobSearchConfig, implements it. type Section interface { // Map returns the section as a map of keys to values, with nested // sections as nested maps. Map() map[string]any // Flatten returns the section as a flat map of dotted keys, each // prefixed with the section's own key (its JSCK* constant), so a // section's keys match how its parent flattens it. JobSearchConfig, the // embedded JSCSourcesSource and types used under several keys, such as // JSCRateLimitRule, have no prefix. Flatten() map[string]any // TypeOf returns the kind of the value at key, which is how it is read // from Viper, or an error for an unknown key. TypeOf(key string) (reflect.Kind, error) // ValueOf returns the value at key, or an error for an unknown key. ValueOf(key string) (any, error) // SetValue stores value at key. value may be of the field's type or a // string to parse (e.g. "10m", "true", "42"). The section's validation, // if any, runs first; on error the field is unchanged. SetValue(key string, value any) error } // validator is implemented by sections that need to reject some values. // key is the field's own key within the section; value has already been // converted to the field's type. type validator interface { validate(key string, value any) error } // field is a section struct field that is part of the configuration (see // fieldsOf). type field struct { key string index []int // path for reflect.Value.FieldByIndex typ reflect.Type section bool def string hasDefault bool } // isSectionList reports whether t is a list of sections, []S for a section // struct S. func isSectionList(t reflect.Type) bool { return t.Kind() == reflect.Slice && t.Elem().Kind() == reflect.Struct && t.Elem() != timeType } // fieldsOf lists the configuration fields of a section type in declaration // order, with the fields of embedded structs promoted into it. func fieldsOf(t reflect.Type) []field { var fields []field for i := 0; i < t.NumField(); i++ { sf := t.Field(i) if sf.Anonymous && sf.Type.Kind() == reflect.Struct { for _, f := range fieldsOf(sf.Type) { f.index = append([]int{i}, f.index...) fields = append(fields, f) } continue } key, ok := sf.Tag.Lookup(keyTag) if !ok || key == "" || key == "-" || !sf.IsExported() { continue } def, hasDefault := sf.Tag.Lookup(defaultTag) fields = append(fields, field{ key: key, index: []int{i}, typ: sf.Type, section: sf.Type.Kind() == reflect.Struct && sf.Type != timeType, def: def, hasDefault: hasDefault, }) } return fields } // structValue dereferences v down to the section struct it holds. func structValue(v any) reflect.Value { rv := reflect.ValueOf(v) for rv.Kind() == reflect.Pointer { rv = rv.Elem() } if rv.Kind() != reflect.Struct { panic(fmt.Sprintf("config: %T is not a configuration section", v)) } return rv } // lookup resolves a dotted key to the leaf field and the section holding it. func lookup(section reflect.Value, key string) (reflect.Value, field, error) { head, rest, nested := strings.Cut(key, ".") for _, f := range fieldsOf(section.Type()) { if f.key != head { continue } switch { case f.section && nested: return lookup(section.FieldByIndex(f.index), rest) case !f.section && !nested: return section, f, nil } // Key matched but the shape didn't: a section with no sub-key, or a // value with one. return reflect.Value{}, field{}, errUnknownKey } return reflect.Value{}, field{}, errUnknownKey } func joinKey(prefix, key string) string { if prefix == "" { return key } return prefix + "." + key } // keysOf returns the keys directly in a section. func keysOf(v any) []string { fields := fieldsOf(structValue(v).Type()) keys := make([]string, 0, len(fields)) for _, f := range fields { keys = append(keys, f.key) } return keys } // mapOf converts a section to a map, with nested sections as nested maps. func mapOf(v any) map[string]any { return mapOfValue(structValue(v)) } // mapOfValue is mapOf for a section struct value. func mapOfValue(section reflect.Value) map[string]any { m := map[string]any{} for _, f := range fieldsOf(section.Type()) { fv := section.FieldByIndex(f.index) if f.section { m[f.key] = mapOfValue(fv) } else { m[f.key] = exportValue(fv) } } return m } // flattenOf converts a section to a flat map of dotted keys, each prefixed // with prefix (if non-empty). func flattenOf(prefix string, v any) map[string]any { m := map[string]any{} flattenInto(m, prefix, structValue(v)) return m } // flattenInto adds section's keys to m, dotted and prefixed with prefix (if // non-empty), descending into nested sections. func flattenInto(m map[string]any, prefix string, section reflect.Value) { for _, f := range fieldsOf(section.Type()) { fv := section.FieldByIndex(f.index) key := joinKey(prefix, f.key) if f.section { flattenInto(m, key, fv) } else { m[key] = exportValue(fv) } } } // exportValue returns a field's value as Map, Flatten and ValueOf give it // out: durations and byte sizes become strings such as "1m30s" and "32KiB", // so values agree with kindOf and can be passed back to SetValue; []string is // copied; a list of sections becomes a []map[string]any (nil if empty); // everything else is unchanged. func exportValue(fv reflect.Value) any { switch fv.Type() { case durationType: return time.Duration(fv.Int()).String() case byteSizeType: return util.ByteSize(fv.Int()).String() } if isSectionList(fv.Type()) { var items []map[string]any for i := range fv.Len() { items = append(items, mapOfValue(fv.Index(i))) } return items } v := fv.Interface() if s, ok := v.([]string); ok { // A copy, so changing it doesn't change the configuration. return slices.Clone(s) } return v } // splitList splits a comma-separated list, trimming spaces around each item // and dropping empty ones, so "" is an empty list. func splitList(s string) []string { var items []string for item := range strings.SplitSeq(s, ",") { if item = strings.TrimSpace(item); item != "" { items = append(items, item) } } return items } // kindOf is the kind reported by TypeOf, i.e. how the value is read from // Viper. Durations and byte sizes are read as strings and parsed by SetValue. func kindOf(t reflect.Type) reflect.Kind { if t == durationType || t == byteSizeType { return reflect.String } return t.Kind() } // typeOf returns the kind of the dotted key in section v (see kindOf), or // errUnknownKey. func typeOf(v any, key string) (reflect.Kind, error) { _, f, err := lookup(structValue(v), key) if err != nil { return reflect.Invalid, err } return kindOf(f.typ), nil } // valueOf returns the value of the dotted key in section v, or errUnknownKey. func valueOf(v any, key string) (any, error) { section, f, err := lookup(structValue(v), key) if err != nil { return nil, err } return exportValue(section.FieldByIndex(f.index)), nil } // setValue sets a dotted key on the section that ptr points to. func setValue(ptr any, key string, value any) error { section, f, err := lookup(structValue(ptr), key) if err != nil { return err } return setField(section, f, value) } // setField converts value to the field's type and, if the section has a // validator, checks it, before setting the field. On error the field is left // unchanged. func setField(section reflect.Value, f field, value any) error { newValue, err := convert(value, f.typ) if err != nil { return err } if v, ok := section.Addr().Interface().(validator); ok { if err := v.validate(f.key, newValue.Interface()); err != nil { return err } } section.FieldByIndex(f.index).Set(newValue) return nil } // convert turns value into a value of type t. Values already of the right // type are used as-is, except that an empty []string becomes nil, as an empty // comma-separated list does (see splitList); strings are parsed. Lists of // sections are converted by convertSectionList. func convert(value any, t reflect.Type) (reflect.Value, error) { if isSectionList(t) { return convertSectionList(value, t) } if value != nil { rv := reflect.ValueOf(value) if rv.Type().AssignableTo(t) { if t == stringSliceType && rv.Len() == 0 { return reflect.Zero(t), nil } return rv, nil } if s, ok := value.(string); ok { return parseString(s, t) } } return reflect.Value{}, fmt.Errorf("unsupported type %T", value) } // convertSectionList turns value into a list of sections of type t: a value // of type t, a list of maps of keys to values, or a JSON array of objects. // nil and empty lists become a nil list. func convertSectionList(value any, t reflect.Type) (reflect.Value, error) { if s, ok := value.(string); ok { var items []map[string]any if strings.TrimSpace(s) != "" { if err := json.Unmarshal([]byte(s), &items); err != nil { return reflect.Value{}, fmt.Errorf("unable to convert to a list of %s: %w", t.Elem(), err) } } value = items } if value == nil { return reflect.Zero(t), nil } rv := reflect.ValueOf(value) if rv.Kind() != reflect.Slice { return reflect.Value{}, fmt.Errorf("unsupported type %T", value) } if rv.Len() == 0 { return reflect.Zero(t), nil } if rv.Type().AssignableTo(t) { for i := range rv.Len() { if err := validateSection(rv.Index(i)); err != nil { return reflect.Value{}, fmt.Errorf("item %d: %w", i, err) } } return rv, nil } list := reflect.MakeSlice(t, 0, rv.Len()) for i := range rv.Len() { item, err := sectionFromMap(rv.Index(i).Interface(), t.Elem()) if err != nil { return reflect.Value{}, fmt.Errorf("item %d: %w", i, err) } list = reflect.Append(list, item) } return list, nil } // sectionFromMap builds a section of type t from a map of its keys to values, // starting from its defaults. Values that are neither strings nor of the // field's type (e.g. numbers read from YAML) are formatted and parsed. The // finished section is validated as a whole (see validateSection), so a key // left out of the map can't skip its validation. func sectionFromMap(value any, t reflect.Type) (reflect.Value, error) { item, ok := value.(map[string]any) if !ok { return reflect.Value{}, fmt.Errorf("unsupported type %T", value) } section := reflect.New(t).Elem() if err := applyDefaultsTo("", section); err != nil { return reflect.Value{}, err } for k, v := range item { s, f, err := lookup(section, k) if err != nil { return reflect.Value{}, fmt.Errorf("%w %q", err, k) } if _, isString := v.(string); v != nil && !isString && !reflect.TypeOf(v).AssignableTo(f.typ) { v = fmt.Sprint(v) } if err := setField(s, f, v); err != nil { return reflect.Value{}, fmt.Errorf("%s: %w", k, err) } } if err := validateSection(section); err != nil { return reflect.Value{}, err } return section, nil } // validateSection runs the section's validator, if it has one, on every value // field directly in it, so values that were never set (defaults or zero // values) are checked too. func validateSection(section reflect.Value) error { v, ok := reflect.New(section.Type()).Interface().(validator) if !ok { return nil } for _, f := range fieldsOf(section.Type()) { if f.section { continue } if err := v.validate(f.key, section.FieldByIndex(f.index).Interface()); err != nil { return fmt.Errorf("%s: %w", f.key, err) } } return nil } // parseString parses s as a value of type t. It supports strings, bools, // signed and unsigned integers, floats, time.Duration, ByteSize and []string // (from a comma-separated list, see splitList). func parseString(s string, t reflect.Type) (reflect.Value, error) { v := reflect.New(t).Elem() switch t { case durationType: d, err := time.ParseDuration(s) if err != nil { return reflect.Value{}, fmt.Errorf("unable to convert to duration: %w", err) } v.SetInt(int64(d)) return v, nil case byteSizeType: b, err := util.ParseByteSize(s) if err != nil { return reflect.Value{}, fmt.Errorf("unable to convert to byte size: %w", err) } v.SetInt(int64(b)) return v, nil case stringSliceType: v.Set(reflect.ValueOf(splitList(s))) return v, nil } var err error switch t.Kind() { case reflect.String: v.SetString(s) case reflect.Bool: var b bool b, err = strconv.ParseBool(s) v.SetBool(b) case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: var n int64 n, err = strconv.ParseInt(s, 10, t.Bits()) v.SetInt(n) case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: var n uint64 n, err = strconv.ParseUint(s, 10, t.Bits()) v.SetUint(n) case reflect.Float32, reflect.Float64: var n float64 n, err = strconv.ParseFloat(s, t.Bits()) v.SetFloat(n) default: return reflect.Value{}, fmt.Errorf("unsupported type string for %s", t) } if err != nil { return reflect.Value{}, fmt.Errorf("unable to convert %q to %s: %w", s, t, err) } return v, nil } // applyDefaults sets every field that has a default tag, recursing into // nested sections. ptr must point to a section struct. func applyDefaults(ptr any) error { return applyDefaultsTo("", structValue(ptr)) } // applyDefaultsTo sets every field in section that has a default tag, // recursing into nested sections. prefix is the section's dotted key, used in // error messages. func applyDefaultsTo(prefix string, section reflect.Value) error { for _, f := range fieldsOf(section.Type()) { key := joinKey(prefix, f.key) if f.section { if err := applyDefaultsTo(key, section.FieldByIndex(f.index)); err != nil { return err } continue } if !f.hasDefault { continue } if err := setField(section, f, f.def); err != nil { return fmt.Errorf("invalid default for %s: %w", key, err) } } return nil }