Fixed releases with 1e9999 ("no limit") being skipped as unreadable

internal/findatender/api.go

@@ -8,6 +8,7 @@ import (
 	"fmt"
 	"io"
 	"log/slog"
+	"math"
 	"regexp"
 	"strconv"
 	"strings"
@@ -65,81 +66,119 @@ type safeReleasePackage struct {
 // dateOnly matches a calendar date with no time of day: "2006-10-02".
 var dateOnly = regexp.MustCompile(`^\d{4}-\d{2}-\d{2}$`)
 
-// unmarshalLenientDates decodes raw json into target like json.Unmarshal, after
-// repairing date fields that time.Time cannot parse.
-func unmarshalLenientDates(raw json.RawMessage, target any) error {
-	fixed, err := repairDates(raw)
+// unmarshalLenient decodes raw json into target like json.Unmarshal, after
+// repairing values that it cannot decode (see repairJSON).
+func unmarshalLenient(raw json.RawMessage, target any) error {
+	fixed, err := repairJSON(raw)
 	if err != nil {
 		return err
 	}
 	return json.Unmarshal(fixed, target)
 }
 
-// repairDates returns a copy of raw in which every date field holding a
-// date-only value ("2006-10-02") is rewritten as midnight UTC
-// ("2006-10-02T00:00:00Z"), and every date field holding an empty string is
-// set to null. Values that already have a time, values in other fields, and
-// dates that appear inside free text are left alone. If nothing needs
-// repairing, raw is returned unchanged.
+// repairJSON returns a copy of raw with the values json.Unmarshal cannot
+// decode repaired:
 //
-// A date field is one whose name is "date" or starts with "date", contains
-// "Date", or ends with "Deadline".
-func repairDates(raw json.RawMessage) (json.RawMessage, error) {
+//   - every date field holding a date-only value ("2006-10-02") is rewritten
+//     as midnight UTC ("2006-10-02T00:00:00Z"), and every date field holding
+//     an empty string is set to null. Values that already have a time, values
+//     in other fields, and dates that appear inside free text are left alone.
+//     A date field is one whose name is "date" or starts with "date",
+//     contains "Date", or ends with "Deadline".
+//   - every number too large for a float64 is replaced by math.MaxInt64 (or
+//     math.MinInt64 if negative). OCDS writes "no limit" as 1e9999, as JSON
+//     has no infinity; math.MaxInt64 still fits the int and float64 fields
+//     it is found in, and is marshalled again where +Inf would not be.
+//
+// If nothing needs repairing, raw is returned unchanged.
+func repairJSON(raw json.RawMessage) (json.RawMessage, error) {
 	decoder := json.NewDecoder(bytes.NewReader(raw))
 	decoder.UseNumber() // keep numbers exactly as written, not as float64
 
 	var doc any
 	if err := decoder.Decode(&doc); err != nil {
-		return nil, fmt.Errorf("repairing dates: %w", err)
+		return nil, fmt.Errorf("repairing json: %w", err)
 	}
 
 	if _, err := decoder.Token(); !errors.Is(err, io.EOF) {
 		return nil, errors.New(
-			"findatender: repairing dates: unexpected data after the JSON value",
+			"findatender: repairing json: unexpected data after the JSON value",
 		)
 	}
 
-	if !recursiveRepairDates(doc) {
+	repaired, changed := recursiveRepair("", doc)
+	if !changed {
 		return raw, nil
 	}
 
-	return json.Marshal(doc)
+	return json.Marshal(repaired)
 }
 
-// recursiveRepairDates fixes date fields in value in place and reports whether
-// it changed anything.
-func recursiveRepairDates(value any) bool {
-	changed := false
+// recursiveRepair repairs value, found in the field named key ("" for an
+// array element or the document itself), and its contents (see repairJSON).
+// Maps and slices are repaired in place; the repaired value is returned, and
+// whether anything changed.
+func recursiveRepair(key string, value any) (any, bool) {
 	switch typed := value.(type) {
 	case map[string]any:
-		for key, fieldValue := range typed {
-			if text, ok := fieldValue.(string); ok && isDateField(key) {
-				switch {
-				case dateOnly.MatchString(text):
-					typed[key] = text + "T00:00:00Z"
-					changed = true
-				case text == "":
-					typed[key] = nil
-					changed = true
-				}
-				continue
-			}
-			if recursiveRepairDates(fieldValue) {
+		changed := false
+		for fieldKey, fieldValue := range typed {
+			repaired, fieldChanged := recursiveRepair(fieldKey, fieldValue)
+			if fieldChanged {
+				typed[fieldKey] = repaired
 				changed = true
 			}
 		}
+		return typed, changed
 	case []any:
-		for _, element := range typed {
-			if recursiveRepairDates(element) {
+		changed := false
+		for i, element := range typed {
+			repaired, elementChanged := recursiveRepair("", element)
+			if elementChanged {
+				typed[i] = repaired
 				changed = true
 			}
 		}
+		return typed, changed
+	case string:
+		return repairDate(key, typed)
+	case json.Number:
+		return repairNumber(typed)
+	default:
+		return value, false
+	}
+}
+
+// repairDate repairs text if key is a date field (see repairJSON).
+func repairDate(key, text string) (any, bool) {
+	if !isDateField(key) {
+		return text, false
+	}
+	switch {
+	case dateOnly.MatchString(text):
+		return text + "T00:00:00Z", true
+	case text == "":
+		return nil, true
+	default:
+		return text, false
+	}
+}
+
+// repairNumber replaces a number too large for a float64 (see repairJSON).
+func repairNumber(number json.Number) (any, bool) {
+	parsed, err := strconv.ParseFloat(number.String(), 64)
+	switch {
+	case !errors.Is(err, strconv.ErrRange) || !math.IsInf(parsed, 0):
+		return number, false
+	case parsed > 0:
+		return json.Number(strconv.FormatInt(math.MaxInt64, 10)), true
+	default:
+		return json.Number(strconv.FormatInt(math.MinInt64, 10)), true
 	}
-	return changed
 }
 
 // isDateField reports whether a field holds a date, by its name (see
-// repairDates).
+// repairJSON).
 func isDateField(name string) bool {
 	return strings.HasPrefix(name, "date") ||
 		strings.Contains(name, "Date") ||
@@ -224,8 +263,8 @@ func missingIDsError(haveOCID, haveID bool) error {
 }
 
 // safeUnpackReleases unpacks an API response into an OCDS ReleasePackage,
-// handling API irregularities (malformed dates, missing fields, invalid
-// releases, etc).
+// handling API irregularities (malformed dates, 1e9999 for no limit, missing
+// fields, invalid releases, etc).
 //
 // This method takes ownership of the response body and closes it when done.
 func safeUnpackReleases(
@@ -265,7 +304,7 @@ func safeUnpackReleases(
 
 	for _, rawRelease := range releasePackage.Releases {
 		var release ocds.Release
-		err := unmarshalLenientDates(rawRelease, &release)
+		err := unmarshalLenient(rawRelease, &release)
 		if err != nil {
 			ocid, releaseID, _ := releaseIDs(rawRelease)
 			logger.Warn(

internal/findatender/api_test.go

@@ -8,6 +8,7 @@ import (
 	"fmt"
 	"io"
 	"log/slog"
+	"math"
 	"net/http"
 	"net/http/httptest"
 	"strings"
@@ -763,7 +764,7 @@ func TestFetchAllRepeatedNextLink(t *testing.T) {
 	}
 }
 
-func TestUnmarshalLenientDates(t *testing.T) {
+func TestUnmarshalLenient(t *testing.T) {
 	raw := json.RawMessage(`{
 		"date": "2006-10-02",
 		"publishedDate": "",
@@ -793,7 +794,7 @@ func TestUnmarshalLenientDates(t *testing.T) {
 		} `json:"awards"`
 		Value json.Number `json:"value"`
 	}
-	if err := unmarshalLenientDates(raw, &got); err != nil {
+	if err := unmarshalLenient(raw, &got); err != nil {
 		t.Fatal(err)
 	}
 
@@ -833,9 +834,77 @@ func TestUnmarshalLenientDates(t *testing.T) {
 	}
 }
 
-func TestRepairDatesLeavesValidJSONAlone(t *testing.T) {
+func TestUnmarshalLenientNumbers(t *testing.T) {
+	tests := []struct {
+		raw       string
+		wantInt   int64
+		wantFloat float64
+	}{
+		{`1e9999`, math.MaxInt64, math.MaxInt64},
+		{`-1e9999`, math.MinInt64, math.MinInt64},
+		{`1E+400`, math.MaxInt64, math.MaxInt64},
+		{`3`, 3, 3},
+	}
+	for _, tc := range tests {
+		t.Run(tc.raw, func(t *testing.T) {
+			var got struct {
+				Int   *int64   `json:"int"`
+				Float *float64 `json:"float"`
+			}
+			raw := json.RawMessage(
+				`{"int": ` + tc.raw + `, "float": ` + tc.raw + `}`,
+			)
+			if err := unmarshalLenient(raw, &got); err != nil {
+				t.Fatal(err)
+			}
+			if *got.Int != tc.wantInt {
+				t.Errorf("int = %d; want %d", *got.Int, tc.wantInt)
+			}
+			if *got.Float != tc.wantFloat {
+				t.Errorf("float = %v; want %v", *got.Float, tc.wantFloat)
+			}
+		})
+	}
+}
+
+// TestUnmarshalLenientNoLimit decodes OCDS's 1e9999 ("no limit") into the
+// release fields Find a Tender sends it in, and checks the release can still
+// be marshalled, as Convert does to store its JSON.
+func TestUnmarshalLenientNoLimit(t *testing.T) {
+	raw := json.RawMessage(`{
+		"ocid": "ocds-1",
+		"id": "r1",
+		"date": "2026-10-16T15:52:30+01:00",
+		"tender": {
+			"lotDetails": {"maximumLotsBidPerSupplier": 1e9999},
+			"techniques": {
+				"frameworkAgreement": {"maximumParticipants": 1e9999}
+			}
+		}
+	}`)
+
+	var release ocds.Release
+	if err := unmarshalLenient(raw, &release); err != nil {
+		t.Fatal(err)
+	}
+	lots := release.Tender.LotDetails.MaximumLotsBidPerSupplier
+	if lots == nil || *lots != math.MaxInt64 {
+		t.Errorf("maximumLotsBidPerSupplier = %v", lots)
+	}
+	framework := release.Tender.Techniques.FrameworkAgreement
+	if participants := framework.MaximumParticipants; participants == nil ||
+		*participants != math.MaxInt64 {
+		t.Errorf("maximumParticipants = %v", participants)
+	}
+
+	if _, err := json.Marshal(release); err != nil {
+		t.Errorf("marshalling the repaired release: %v", err)
+	}
+}
+
+func TestRepairJSONLeavesValidJSONAlone(t *testing.T) {
 	raw := json.RawMessage(`{"date": "2006-10-02T15:33:02+07:00", "title": "x"}`)
-	out, err := repairDates(raw)
+	out, err := repairJSON(raw)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -844,15 +913,15 @@ func TestRepairDatesLeavesValidJSONAlone(t *testing.T) {
 	}
 }
 
-func TestRepairDatesRejectsBadJSON(t *testing.T) {
+func TestRepairJSONRejectsBadJSON(t *testing.T) {
 	for _, raw := range []string{`{`, `{"date": "2006-10-02"} trailing`} {
-		if _, err := repairDates(json.RawMessage(raw)); err == nil {
+		if _, err := repairJSON(json.RawMessage(raw)); err == nil {
 			t.Errorf("expected an error for %q", raw)
 		}
 	}
 }
 
-func TestUnmarshalLenientDatesErrors(t *testing.T) {
+func TestUnmarshalLenientErrors(t *testing.T) {
 	var v struct {
 		Date time.Time `json:"date"`
 	}
@@ -864,7 +933,7 @@ func TestUnmarshalLenientDatesErrors(t *testing.T) {
 	}
 	for name, raw := range tests {
 		t.Run(name, func(t *testing.T) {
-			if err := unmarshalLenientDates(json.RawMessage(raw), &v); err == nil {
+			if err := unmarshalLenient(json.RawMessage(raw), &v); err == nil {
 				t.Errorf("expected an error for %s", raw)
 			}
 		})