package config import ( "encoding/json" "errors" "fmt" "reflect" "slices" "strconv" "strings" "time" "git.avsdev.uk/Go/job-search/internal/util" ) // Configuration sections are plain structs described with struct tags: // // cfg:"key" the field's configuration key. Fields without it are // not part of the configuration. // default:"value" the field's default, parsed the same way as a string // passed to SetValue (e.g. "10m", "true", "42"). // // A tagged field whose type is a struct (other than time.Time) is a nested // section, addressed as "section.key". An embedded struct has its tagged // fields promoted into the enclosing section, just as Go promotes fields. // // time.Duration fields report reflect.String from TypeOf and accept either a // time.Duration or a duration string such as "90s" in SetValue. // util.ByteSize fields work the same way, with size strings such as "32KiB" // (or "-1" for no limit). // // []string fields report reflect.Slice and accept either a []string or a // comma-separated string such as "a, b" in SetValue (and so in default tags). // An empty list is stored as nil. // // A list of sections, []S where S is a section struct (e.g. a list of extra // headers), is a single key reporting reflect.Slice. It is given out as a // []map[string]any, one map per item as Map gives it, and SetValue accepts // a []S, a list of maps (as Viper reads a YAML or JSON list of objects) or a // JSON array string. Each item starts from its defaults; its keys are set as // SetValue sets them, so values may be strings to parse. If the item type has // a validator, every field of each item is validated, including ones not set. // // A section can check values before they are stored by implementing // validator on its pointer type. // The struct tags section.go reads. const ( // keyTag holds a field's configuration key. keyTag = "cfg" // defaultTag holds a field's default value. defaultTag = "default" ) // The unknown key error and the field types section.go handles specially. var ( // errUnknownKey is returned for a key that names no field. errUnknownKey = errors.New("config: unknown configuration key") // durationType is time.Duration, read and given out as a string. durationType = reflect.TypeOf(time.Duration(0)) // byteSizeType is util.ByteSize, read and given out as a string. byteSizeType = reflect.TypeOf(util.ByteSize(0)) // timeType is time.Time, a struct that is a value, not a section. timeType = reflect.TypeOf(time.Time{}) // stringSliceType is []string, set from a comma-separated list. stringSliceType = reflect.TypeOf([]string{}) ) // Section is the method set of every configuration section, including // JobSearchConfig itself, implemented on the section's pointer type. Keys are // dotted paths relative to the section, e.g. "sleeper.interval" in JSCSources. // Durations and byte sizes are given out as strings such as "10m0s" and // "32KiB", matching TypeOf's reflect.String for them. TestSections checks every // section, found by walking JobSearchConfig, implements it. type Section interface { // Map returns the section as a map of keys to values, with nested // sections as nested maps. Map() map[string]any // Flatten returns the section as a flat map of dotted keys, each // prefixed with the section's own key (its JSCK* constant), so a // section's keys match how its parent flattens it. JobSearchConfig, the // embedded JSCSourcesSource and types used under several keys, such as // JSCRateLimitRule, have no prefix. Flatten() map[string]any // TypeOf returns the kind of the value at key, which is how it is read // from Viper, or an error for an unknown key. TypeOf(key string) (reflect.Kind, error) // ValueOf returns the value at key, or an error for an unknown key. ValueOf(key string) (any, error) // SetValue stores value at key. value may be of the field's type or a // string to parse (e.g. "10m", "true", "42"). The section's validation, // if any, runs first; on error the field is unchanged. SetValue(key string, value any) error } // validator is implemented by sections that need to reject some values. // key is the field's own key within the section; value has already been // converted to the field's type. type validator interface { validate(key string, value any) error } // field is a section struct field that is part of the configuration (see // fieldsOf). type field struct { key string index []int // path for reflect.Value.FieldByIndex fieldType reflect.Type section bool defaultValue string hasDefault bool } // isSectionList reports whether listType is a list of sections, []S for a // section struct S. func isSectionList(listType reflect.Type) bool { return listType.Kind() == reflect.Slice && listType.Elem().Kind() == reflect.Struct && listType.Elem() != timeType } // fieldsOf lists the configuration fields of a section type in declaration // order, with the fields of embedded structs promoted into it. func fieldsOf(structType reflect.Type) []field { var fields []field for i := 0; i < structType.NumField(); i++ { structField := structType.Field(i) if structField.Anonymous && structField.Type.Kind() == reflect.Struct { for _, embeddedField := range fieldsOf(structField.Type) { embeddedField.index = append([]int{i}, embeddedField.index...) fields = append(fields, embeddedField) } continue } key, ok := structField.Tag.Lookup(keyTag) if !ok || key == "" || key == "-" || !structField.IsExported() { continue } defaultValue, hasDefault := structField.Tag.Lookup(defaultTag) fields = append(fields, field{ key: key, index: []int{i}, fieldType: structField.Type, section: structField.Type.Kind() == reflect.Struct && structField.Type != timeType, defaultValue: defaultValue, hasDefault: hasDefault, }) } return fields } // structValue dereferences value down to the section struct it holds. func structValue(value any) reflect.Value { reflected := reflect.ValueOf(value) for reflected.Kind() == reflect.Pointer { reflected = reflected.Elem() } if reflected.Kind() != reflect.Struct { panic(fmt.Sprintf("config: %T is not a configuration section", value)) } return reflected } // lookup resolves a dotted key to the leaf field and the section holding it. func lookup(section reflect.Value, key string) (reflect.Value, field, error) { head, rest, nested := strings.Cut(key, ".") for _, configField := range fieldsOf(section.Type()) { if configField.key != head { continue } switch { case configField.section && nested: return lookup(section.FieldByIndex(configField.index), rest) case !configField.section && !nested: return section, configField, nil } // Key matched but the shape didn't: a section with no sub-key, or a // value with one. return reflect.Value{}, field{}, errUnknownKey } return reflect.Value{}, field{}, errUnknownKey } func joinKey(prefix, key string) string { if prefix == "" { return key } return prefix + "." + key } // keysOf returns the keys directly in a section. func keysOf(value any) []string { fields := fieldsOf(structValue(value).Type()) keys := make([]string, 0, len(fields)) for _, configField := range fields { keys = append(keys, configField.key) } return keys } // mapOf converts a section to a map, with nested sections as nested maps. func mapOf(value any) map[string]any { return mapOfValue(structValue(value)) } // mapOfValue is mapOf for a section struct value. func mapOfValue(section reflect.Value) map[string]any { values := map[string]any{} for _, configField := range fieldsOf(section.Type()) { fieldValue := section.FieldByIndex(configField.index) if configField.section { values[configField.key] = mapOfValue(fieldValue) } else { values[configField.key] = exportValue(fieldValue) } } return values } // flattenOf converts a section to a flat map of dotted keys, each prefixed // with prefix (if non-empty). func flattenOf(prefix string, value any) map[string]any { flat := map[string]any{} flattenInto(flat, prefix, structValue(value)) return flat } // flattenInto adds section's keys to m, dotted and prefixed with prefix (if // non-empty), descending into nested sections. func flattenInto(flat map[string]any, prefix string, section reflect.Value) { for _, configField := range fieldsOf(section.Type()) { fieldValue := section.FieldByIndex(configField.index) key := joinKey(prefix, configField.key) if configField.section { flattenInto(flat, key, fieldValue) } else { flat[key] = exportValue(fieldValue) } } } // exportValue returns a field's value as Map, Flatten and ValueOf give it // out: durations and byte sizes become strings such as "1m30s" and "32KiB", // so values agree with kindOf and can be passed back to SetValue; []string is // copied; a list of sections becomes a []map[string]any (nil if empty); // everything else is unchanged. func exportValue(fieldValue reflect.Value) any { switch fieldValue.Type() { case durationType: return time.Duration(fieldValue.Int()).String() case byteSizeType: return util.ByteSize(fieldValue.Int()).String() } if isSectionList(fieldValue.Type()) { var items []map[string]any for i := range fieldValue.Len() { items = append(items, mapOfValue(fieldValue.Index(i))) } return items } value := fieldValue.Interface() if list, ok := value.([]string); ok { // A copy, so changing it doesn't change the configuration. return slices.Clone(list) } return value } // splitList splits a comma-separated list, trimming spaces around each item // and dropping empty ones, so "" is an empty list. func splitList(list string) []string { var items []string for item := range strings.SplitSeq(list, ",") { if item = strings.TrimSpace(item); item != "" { items = append(items, item) } } return items } // kindOf is the kind reported by TypeOf, i.e. how the value is read from // Viper. Durations and byte sizes are read as strings and parsed by SetValue. func kindOf(fieldType reflect.Type) reflect.Kind { if fieldType == durationType || fieldType == byteSizeType { return reflect.String } return fieldType.Kind() } // typeOf returns the kind of the dotted key in the section value (see // kindOf), or errUnknownKey. func typeOf(value any, key string) (reflect.Kind, error) { _, configField, err := lookup(structValue(value), key) if err != nil { return reflect.Invalid, err } return kindOf(configField.fieldType), nil } // valueOf returns the value of the dotted key in the section value, or // errUnknownKey. func valueOf(value any, key string) (any, error) { section, configField, err := lookup(structValue(value), key) if err != nil { return nil, err } return exportValue(section.FieldByIndex(configField.index)), nil } // setValue sets a dotted key on the section that ptr points to. func setValue(ptr any, key string, value any) error { section, configField, err := lookup(structValue(ptr), key) if err != nil { return err } return setField(section, configField, value) } // setField converts value to the field's type and, if the section has a // validator, checks it, before setting the field. On error the field is left // unchanged. func setField(section reflect.Value, configField field, value any) error { newValue, err := convert(value, configField.fieldType) if err != nil { return err } if checker, ok := section.Addr().Interface().(validator); ok { err := checker.validate(configField.key, newValue.Interface()) if err != nil { return err } } section.FieldByIndex(configField.index).Set(newValue) return nil } // convert turns value into a value of type targetType. Values already of the // right type are used as-is, except that an empty []string becomes nil, as an // empty comma-separated list does (see splitList); strings are parsed. Lists of // sections are converted by convertSectionList. func convert(value any, targetType reflect.Type) (reflect.Value, error) { if isSectionList(targetType) { return convertSectionList(value, targetType) } if value != nil { reflected := reflect.ValueOf(value) if reflected.Type().AssignableTo(targetType) { if targetType == stringSliceType && reflected.Len() == 0 { return reflect.Zero(targetType), nil } return reflected, nil } if text, ok := value.(string); ok { return parseString(text, targetType) } } return reflect.Value{}, fmt.Errorf("config: unsupported type %T", value) } // convertSectionList turns value into a list of sections of type listType: a // value of type listType, a list of maps of keys to values, or a JSON array of // objects. nil and empty lists become a nil list. func convertSectionList( value any, listType reflect.Type, ) (reflect.Value, error) { if text, ok := value.(string); ok { var items []map[string]any if strings.TrimSpace(text) != "" { if err := json.Unmarshal([]byte(text), &items); err != nil { return reflect.Value{}, fmt.Errorf( "unable to convert to a list of %s: %w", listType.Elem(), err, ) } } value = items } if value == nil { return reflect.Zero(listType), nil } reflected := reflect.ValueOf(value) if reflected.Kind() != reflect.Slice { return reflect.Value{}, fmt.Errorf("config: unsupported type %T", value) } if reflected.Len() == 0 { return reflect.Zero(listType), nil } if reflected.Type().AssignableTo(listType) { for i := range reflected.Len() { if err := validateSection(reflected.Index(i)); err != nil { return reflect.Value{}, fmt.Errorf("item %d: %w", i, err) } } return reflected, nil } list := reflect.MakeSlice(listType, 0, reflected.Len()) for i := range reflected.Len() { item, err := sectionFromMap(reflected.Index(i).Interface(), listType.Elem()) if err != nil { return reflect.Value{}, fmt.Errorf("item %d: %w", i, err) } list = reflect.Append(list, item) } return list, nil } // sectionFromMap builds a section of type structType from value, a map of its // keys to values, starting from its defaults. Values that are neither strings // nor of the field's type (e.g. numbers read from YAML) are formatted and // parsed. The finished section is validated as a whole (see // validateSection), so a key left out of the map can't skip its validation. func sectionFromMap(value any, structType reflect.Type) (reflect.Value, error) { item, ok := value.(map[string]any) if !ok { return reflect.Value{}, fmt.Errorf("config: unsupported type %T", value) } section := reflect.New(structType).Elem() if err := applyDefaultsTo("", section); err != nil { return reflect.Value{}, err } for itemKey, itemValue := range item { holder, configField, err := lookup(section, itemKey) if err != nil { return reflect.Value{}, fmt.Errorf("%w %q", err, itemKey) } if _, isString := itemValue.(string); itemValue != nil && !isString && !reflect.TypeOf(itemValue).AssignableTo(configField.fieldType) { itemValue = fmt.Sprint(itemValue) } if err := setField(holder, configField, itemValue); err != nil { return reflect.Value{}, fmt.Errorf("%s: %w", itemKey, err) } } if err := validateSection(section); err != nil { return reflect.Value{}, err } return section, nil } // validateSection runs the section's validator, if it has one, on every value // field directly in it, so values that were never set (defaults or zero // values) are checked too. func validateSection(section reflect.Value) error { checker, ok := reflect.New(section.Type()).Interface().(validator) if !ok { return nil } for _, configField := range fieldsOf(section.Type()) { if configField.section { continue } value := section.FieldByIndex(configField.index).Interface() if err := checker.validate(configField.key, value); err != nil { return fmt.Errorf("%s: %w", configField.key, err) } } return nil } // parseString parses text as a value of type targetType. It supports strings, // bools, signed and unsigned integers, floats, time.Duration, ByteSize and // []string (from a comma-separated list, see splitList). func parseString(text string, targetType reflect.Type) (reflect.Value, error) { parsed := reflect.New(targetType).Elem() switch targetType { case durationType: duration, err := time.ParseDuration(text) if err != nil { return reflect.Value{}, fmt.Errorf( "unable to convert to duration: %w", err, ) } parsed.SetInt(int64(duration)) return parsed, nil case byteSizeType: size, err := util.ParseByteSize(text) if err != nil { return reflect.Value{}, fmt.Errorf( "unable to convert to byte size: %w", err, ) } parsed.SetInt(int64(size)) return parsed, nil case stringSliceType: parsed.Set(reflect.ValueOf(splitList(text))) return parsed, nil } var err error switch targetType.Kind() { case reflect.String: parsed.SetString(text) case reflect.Bool: var flag bool flag, err = strconv.ParseBool(text) parsed.SetBool(flag) case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: var number int64 number, err = strconv.ParseInt(text, 10, targetType.Bits()) parsed.SetInt(number) case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: var number uint64 number, err = strconv.ParseUint(text, 10, targetType.Bits()) parsed.SetUint(number) case reflect.Float32, reflect.Float64: var number float64 number, err = strconv.ParseFloat(text, targetType.Bits()) parsed.SetFloat(number) default: return reflect.Value{}, fmt.Errorf( "config: unsupported type string for %s", targetType, ) } if err != nil { return reflect.Value{}, fmt.Errorf( "unable to convert %q to %s: %w", text, targetType, err, ) } return parsed, nil } // applyDefaults sets every field that has a default tag, recursing into // nested sections. ptr must point to a section struct. func applyDefaults(ptr any) error { return applyDefaultsTo("", structValue(ptr)) } // applyDefaultsTo sets every field in section that has a default tag, // recursing into nested sections. prefix is the section's dotted key, used in // error messages. func applyDefaultsTo(prefix string, section reflect.Value) error { for _, configField := range fieldsOf(section.Type()) { key := joinKey(prefix, configField.key) if configField.section { nested := section.FieldByIndex(configField.index) if err := applyDefaultsTo(key, nested); err != nil { return err } continue } if !configField.hasDefault { continue } err := setField(section, configField, configField.defaultValue) if err != nil { return fmt.Errorf("invalid default for %s: %w", key, err) } } return nil }