diff --git a/internal/findatender/api.go b/internal/findatender/api.go index a05bb23..a50a48f 100644 --- a/internal/findatender/api.go +++ b/internal/findatender/api.go @@ -8,6 +8,7 @@ import ( "fmt" "io" "log/slog" + "math" "regexp" "strconv" "strings" @@ -65,81 +66,119 @@ type safeReleasePackage struct { // dateOnly matches a calendar date with no time of day: "2006-10-02". var dateOnly = regexp.MustCompile(`^\d{4}-\d{2}-\d{2}$`) -// unmarshalLenientDates decodes raw json into target like json.Unmarshal, after -// repairing date fields that time.Time cannot parse. -func unmarshalLenientDates(raw json.RawMessage, target any) error { - fixed, err := repairDates(raw) +// unmarshalLenient decodes raw json into target like json.Unmarshal, after +// repairing values that it cannot decode (see repairJSON). +func unmarshalLenient(raw json.RawMessage, target any) error { + fixed, err := repairJSON(raw) if err != nil { return err } return json.Unmarshal(fixed, target) } -// repairDates returns a copy of raw in which every date field holding a -// date-only value ("2006-10-02") is rewritten as midnight UTC -// ("2006-10-02T00:00:00Z"), and every date field holding an empty string is -// set to null. Values that already have a time, values in other fields, and -// dates that appear inside free text are left alone. If nothing needs -// repairing, raw is returned unchanged. +// repairJSON returns a copy of raw with the values json.Unmarshal cannot +// decode repaired: // -// A date field is one whose name is "date" or starts with "date", contains -// "Date", or ends with "Deadline". -func repairDates(raw json.RawMessage) (json.RawMessage, error) { +// - every date field holding a date-only value ("2006-10-02") is rewritten +// as midnight UTC ("2006-10-02T00:00:00Z"), and every date field holding +// an empty string is set to null. Values that already have a time, values +// in other fields, and dates that appear inside free text are left alone. +// A date field is one whose name is "date" or starts with "date", +// contains "Date", or ends with "Deadline". +// - every number too large for a float64 is replaced by math.MaxInt64 (or +// math.MinInt64 if negative). OCDS writes "no limit" as 1e9999, as JSON +// has no infinity; math.MaxInt64 still fits the int and float64 fields +// it is found in, and is marshalled again where +Inf would not be. +// +// If nothing needs repairing, raw is returned unchanged. +func repairJSON(raw json.RawMessage) (json.RawMessage, error) { decoder := json.NewDecoder(bytes.NewReader(raw)) decoder.UseNumber() // keep numbers exactly as written, not as float64 var doc any if err := decoder.Decode(&doc); err != nil { - return nil, fmt.Errorf("repairing dates: %w", err) + return nil, fmt.Errorf("repairing json: %w", err) } if _, err := decoder.Token(); !errors.Is(err, io.EOF) { return nil, errors.New( - "findatender: repairing dates: unexpected data after the JSON value", + "findatender: repairing json: unexpected data after the JSON value", ) } - if !recursiveRepairDates(doc) { + repaired, changed := recursiveRepair("", doc) + if !changed { return raw, nil } - return json.Marshal(doc) + return json.Marshal(repaired) } -// recursiveRepairDates fixes date fields in value in place and reports whether -// it changed anything. -func recursiveRepairDates(value any) bool { - changed := false +// recursiveRepair repairs value, found in the field named key ("" for an +// array element or the document itself), and its contents (see repairJSON). +// Maps and slices are repaired in place; the repaired value is returned, and +// whether anything changed. +func recursiveRepair(key string, value any) (any, bool) { switch typed := value.(type) { case map[string]any: - for key, fieldValue := range typed { - if text, ok := fieldValue.(string); ok && isDateField(key) { - switch { - case dateOnly.MatchString(text): - typed[key] = text + "T00:00:00Z" - changed = true - case text == "": - typed[key] = nil - changed = true - } - continue - } - if recursiveRepairDates(fieldValue) { + changed := false + for fieldKey, fieldValue := range typed { + repaired, fieldChanged := recursiveRepair(fieldKey, fieldValue) + if fieldChanged { + typed[fieldKey] = repaired changed = true } } + return typed, changed case []any: - for _, element := range typed { - if recursiveRepairDates(element) { + changed := false + for i, element := range typed { + repaired, elementChanged := recursiveRepair("", element) + if elementChanged { + typed[i] = repaired changed = true } } + return typed, changed + case string: + return repairDate(key, typed) + case json.Number: + return repairNumber(typed) + default: + return value, false + } +} + +// repairDate repairs text if key is a date field (see repairJSON). +func repairDate(key, text string) (any, bool) { + if !isDateField(key) { + return text, false + } + switch { + case dateOnly.MatchString(text): + return text + "T00:00:00Z", true + case text == "": + return nil, true + default: + return text, false + } +} + +// repairNumber replaces a number too large for a float64 (see repairJSON). +func repairNumber(number json.Number) (any, bool) { + parsed, err := strconv.ParseFloat(number.String(), 64) + switch { + case !errors.Is(err, strconv.ErrRange) || !math.IsInf(parsed, 0): + return number, false + case parsed > 0: + return json.Number(strconv.FormatInt(math.MaxInt64, 10)), true + default: + return json.Number(strconv.FormatInt(math.MinInt64, 10)), true } - return changed } // isDateField reports whether a field holds a date, by its name (see -// repairDates). +// repairJSON). func isDateField(name string) bool { return strings.HasPrefix(name, "date") || strings.Contains(name, "Date") || @@ -224,8 +263,8 @@ func missingIDsError(haveOCID, haveID bool) error { } // safeUnpackReleases unpacks an API response into an OCDS ReleasePackage, -// handling API irregularities (malformed dates, missing fields, invalid -// releases, etc). +// handling API irregularities (malformed dates, 1e9999 for no limit, missing +// fields, invalid releases, etc). // // This method takes ownership of the response body and closes it when done. func safeUnpackReleases( @@ -265,7 +304,7 @@ func safeUnpackReleases( for _, rawRelease := range releasePackage.Releases { var release ocds.Release - err := unmarshalLenientDates(rawRelease, &release) + err := unmarshalLenient(rawRelease, &release) if err != nil { ocid, releaseID, _ := releaseIDs(rawRelease) logger.Warn( diff --git a/internal/findatender/api_test.go b/internal/findatender/api_test.go index f69980c..465e64e 100644 --- a/internal/findatender/api_test.go +++ b/internal/findatender/api_test.go @@ -8,6 +8,7 @@ import ( "fmt" "io" "log/slog" + "math" "net/http" "net/http/httptest" "strings" @@ -763,7 +764,7 @@ func TestFetchAllRepeatedNextLink(t *testing.T) { } } -func TestUnmarshalLenientDates(t *testing.T) { +func TestUnmarshalLenient(t *testing.T) { raw := json.RawMessage(`{ "date": "2006-10-02", "publishedDate": "", @@ -793,7 +794,7 @@ func TestUnmarshalLenientDates(t *testing.T) { } `json:"awards"` Value json.Number `json:"value"` } - if err := unmarshalLenientDates(raw, &got); err != nil { + if err := unmarshalLenient(raw, &got); err != nil { t.Fatal(err) } @@ -833,9 +834,77 @@ func TestUnmarshalLenientDates(t *testing.T) { } } -func TestRepairDatesLeavesValidJSONAlone(t *testing.T) { +func TestUnmarshalLenientNumbers(t *testing.T) { + tests := []struct { + raw string + wantInt int64 + wantFloat float64 + }{ + {`1e9999`, math.MaxInt64, math.MaxInt64}, + {`-1e9999`, math.MinInt64, math.MinInt64}, + {`1E+400`, math.MaxInt64, math.MaxInt64}, + {`3`, 3, 3}, + } + for _, tc := range tests { + t.Run(tc.raw, func(t *testing.T) { + var got struct { + Int *int64 `json:"int"` + Float *float64 `json:"float"` + } + raw := json.RawMessage( + `{"int": ` + tc.raw + `, "float": ` + tc.raw + `}`, + ) + if err := unmarshalLenient(raw, &got); err != nil { + t.Fatal(err) + } + if *got.Int != tc.wantInt { + t.Errorf("int = %d; want %d", *got.Int, tc.wantInt) + } + if *got.Float != tc.wantFloat { + t.Errorf("float = %v; want %v", *got.Float, tc.wantFloat) + } + }) + } +} + +// TestUnmarshalLenientNoLimit decodes OCDS's 1e9999 ("no limit") into the +// release fields Find a Tender sends it in, and checks the release can still +// be marshalled, as Convert does to store its JSON. +func TestUnmarshalLenientNoLimit(t *testing.T) { + raw := json.RawMessage(`{ + "ocid": "ocds-1", + "id": "r1", + "date": "2026-10-16T15:52:30+01:00", + "tender": { + "lotDetails": {"maximumLotsBidPerSupplier": 1e9999}, + "techniques": { + "frameworkAgreement": {"maximumParticipants": 1e9999} + } + } + }`) + + var release ocds.Release + if err := unmarshalLenient(raw, &release); err != nil { + t.Fatal(err) + } + lots := release.Tender.LotDetails.MaximumLotsBidPerSupplier + if lots == nil || *lots != math.MaxInt64 { + t.Errorf("maximumLotsBidPerSupplier = %v", lots) + } + framework := release.Tender.Techniques.FrameworkAgreement + if participants := framework.MaximumParticipants; participants == nil || + *participants != math.MaxInt64 { + t.Errorf("maximumParticipants = %v", participants) + } + + if _, err := json.Marshal(release); err != nil { + t.Errorf("marshalling the repaired release: %v", err) + } +} + +func TestRepairJSONLeavesValidJSONAlone(t *testing.T) { raw := json.RawMessage(`{"date": "2006-10-02T15:33:02+07:00", "title": "x"}`) - out, err := repairDates(raw) + out, err := repairJSON(raw) if err != nil { t.Fatal(err) } @@ -844,15 +913,15 @@ func TestRepairDatesLeavesValidJSONAlone(t *testing.T) { } } -func TestRepairDatesRejectsBadJSON(t *testing.T) { +func TestRepairJSONRejectsBadJSON(t *testing.T) { for _, raw := range []string{`{`, `{"date": "2006-10-02"} trailing`} { - if _, err := repairDates(json.RawMessage(raw)); err == nil { + if _, err := repairJSON(json.RawMessage(raw)); err == nil { t.Errorf("expected an error for %q", raw) } } } -func TestUnmarshalLenientDatesErrors(t *testing.T) { +func TestUnmarshalLenientErrors(t *testing.T) { var v struct { Date time.Time `json:"date"` } @@ -864,7 +933,7 @@ func TestUnmarshalLenientDatesErrors(t *testing.T) { } for name, raw := range tests { t.Run(name, func(t *testing.T) { - if err := unmarshalLenientDates(json.RawMessage(raw), &v); err == nil { + if err := unmarshalLenient(json.RawMessage(raw), &v); err == nil { t.Errorf("expected an error for %s", raw) } })