Fixed releases with 1e9999 ("no limit") being skipped as unreadable

internal/findatender/api.go

@@ -8,6 +8,7 @@ import (
 	"fmt"
 	"io"
 	"log/slog"
+	"math"
 	"regexp"
 	"strconv"
 	"strings"
@@ -65,81 +66,119 @@ type safeReleasePackage struct {
 // dateOnly matches a calendar date with no time of day: "2006-10-02".
 var dateOnly = regexp.MustCompile(`^\d{4}-\d{2}-\d{2}$`)
 
-// unmarshalLenientDates decodes raw json into target like json.Unmarshal, after
-// repairing date fields that time.Time cannot parse.
-func unmarshalLenientDates(raw json.RawMessage, target any) error {
-	fixed, err := repairDates(raw)
+// unmarshalLenient decodes raw json into target like json.Unmarshal, after
+// repairing values that it cannot decode (see repairJSON).
+func unmarshalLenient(raw json.RawMessage, target any) error {
+	fixed, err := repairJSON(raw)
 	if err != nil {
 		return err
 	}
 	return json.Unmarshal(fixed, target)
 }
 
-// repairDates returns a copy of raw in which every date field holding a
-// date-only value ("2006-10-02") is rewritten as midnight UTC
-// ("2006-10-02T00:00:00Z"), and every date field holding an empty string is
-// set to null. Values that already have a time, values in other fields, and
-// dates that appear inside free text are left alone. If nothing needs
-// repairing, raw is returned unchanged.
+// repairJSON returns a copy of raw with the values json.Unmarshal cannot
+// decode repaired:
 //
-// A date field is one whose name is "date" or starts with "date", contains
-// "Date", or ends with "Deadline".
-func repairDates(raw json.RawMessage) (json.RawMessage, error) {
+//   - every date field holding a date-only value ("2006-10-02") is rewritten
+//     as midnight UTC ("2006-10-02T00:00:00Z"), and every date field holding
+//     an empty string is set to null. Values that already have a time, values
+//     in other fields, and dates that appear inside free text are left alone.
+//     A date field is one whose name is "date" or starts with "date",
+//     contains "Date", or ends with "Deadline".
+//   - every number too large for a float64 is replaced by math.MaxInt64 (or
+//     math.MinInt64 if negative). OCDS writes "no limit" as 1e9999, as JSON
+//     has no infinity; math.MaxInt64 still fits the int and float64 fields
+//     it is found in, and is marshalled again where +Inf would not be.
+//
+// If nothing needs repairing, raw is returned unchanged.
+func repairJSON(raw json.RawMessage) (json.RawMessage, error) {
 	decoder := json.NewDecoder(bytes.NewReader(raw))
 	decoder.UseNumber() // keep numbers exactly as written, not as float64
 
 	var doc any
 	if err := decoder.Decode(&doc); err != nil {
-		return nil, fmt.Errorf("repairing dates: %w", err)
+		return nil, fmt.Errorf("repairing json: %w", err)
 	}
 
 	if _, err := decoder.Token(); !errors.Is(err, io.EOF) {
 		return nil, errors.New(
-			"findatender: repairing dates: unexpected data after the JSON value",
+			"findatender: repairing json: unexpected data after the JSON value",
 		)
 	}
 
-	if !recursiveRepairDates(doc) {
+	repaired, changed := recursiveRepair("", doc)
+	if !changed {
 		return raw, nil
 	}
 
-	return json.Marshal(doc)
+	return json.Marshal(repaired)
 }
 
-// recursiveRepairDates fixes date fields in value in place and reports whether
-// it changed anything.
-func recursiveRepairDates(value any) bool {
-	changed := false
+// recursiveRepair repairs value, found in the field named key ("" for an
+// array element or the document itself), and its contents (see repairJSON).
+// Maps and slices are repaired in place; the repaired value is returned, and
+// whether anything changed.
+func recursiveRepair(key string, value any) (any, bool) {
 	switch typed := value.(type) {
 	case map[string]any:
-		for key, fieldValue := range typed {
-			if text, ok := fieldValue.(string); ok && isDateField(key) {
-				switch {
-				case dateOnly.MatchString(text):
-					typed[key] = text + "T00:00:00Z"
-					changed = true
-				case text == "":
-					typed[key] = nil
-					changed = true
-				}
-				continue
-			}
-			if recursiveRepairDates(fieldValue) {
+		changed := false
+		for fieldKey, fieldValue := range typed {
+			repaired, fieldChanged := recursiveRepair(fieldKey, fieldValue)
+			if fieldChanged {
+				typed[fieldKey] = repaired
 				changed = true
 			}
 		}
+		return typed, changed
 	case []any:
-		for _, element := range typed {
-			if recursiveRepairDates(element) {
+		changed := false
+		for i, element := range typed {
+			repaired, elementChanged := recursiveRepair("", element)
+			if elementChanged {
+				typed[i] = repaired
 				changed = true
 			}
 		}
+		return typed, changed
+	case string:
+		return repairDate(key, typed)
+	case json.Number:
+		return repairNumber(typed)
+	default:
+		return value, false
+	}
+}
+
+// repairDate repairs text if key is a date field (see repairJSON).
+func repairDate(key, text string) (any, bool) {
+	if !isDateField(key) {
+		return text, false
+	}
+	switch {
+	case dateOnly.MatchString(text):
+		return text + "T00:00:00Z", true
+	case text == "":
+		return nil, true
+	default:
+		return text, false
+	}
+}
+
+// repairNumber replaces a number too large for a float64 (see repairJSON).
+func repairNumber(number json.Number) (any, bool) {
+	parsed, err := strconv.ParseFloat(number.String(), 64)
+	switch {
+	case !errors.Is(err, strconv.ErrRange) || !math.IsInf(parsed, 0):
+		return number, false
+	case parsed > 0:
+		return json.Number(strconv.FormatInt(math.MaxInt64, 10)), true
+	default:
+		return json.Number(strconv.FormatInt(math.MinInt64, 10)), true
 	}
-	return changed
 }
 
 // isDateField reports whether a field holds a date, by its name (see
-// repairDates).
+// repairJSON).
 func isDateField(name string) bool {
 	return strings.HasPrefix(name, "date") ||
 		strings.Contains(name, "Date") ||
@@ -224,8 +263,8 @@ func missingIDsError(haveOCID, haveID bool) error {
 }
 
 // safeUnpackReleases unpacks an API response into an OCDS ReleasePackage,
-// handling API irregularities (malformed dates, missing fields, invalid
-// releases, etc).
+// handling API irregularities (malformed dates, 1e9999 for no limit, missing
+// fields, invalid releases, etc).
 //
 // This method takes ownership of the response body and closes it when done.
 func safeUnpackReleases(
@@ -265,7 +304,7 @@ func safeUnpackReleases(
 
 	for _, rawRelease := range releasePackage.Releases {
 		var release ocds.Release
-		err := unmarshalLenientDates(rawRelease, &release)
+		err := unmarshalLenient(rawRelease, &release)
 		if err != nil {
 			ocid, releaseID, _ := releaseIDs(rawRelease)
 			logger.Warn(