package findATender import ( "bufio" "bytes" "io" "regexp" ) // Global regex to identify and catch date-only fields var invalidDateRegex = regexp.MustCompile(`:\s*"(\d{4}-\d{2}-\d{2})"\s*([,}\s])`) var invalidPublishedDateRegex = regexp.MustCompile(`"publishedDate":\s*""`) // The UK Government FindATender service doesn't strictly follow OCDS date // format. Sometimes it provides dates only, sometimes it sends empty strings // instead of nulls. type FindATenderStreamCleaner struct { reader *bufio.Reader buffer bytes.Buffer } func NewOCDSStreamCleaner(r io.Reader) *FindATenderStreamCleaner { return &FindATenderStreamCleaner{reader: bufio.NewReader(r)} } func (sc *FindATenderStreamCleaner) Read(p []byte) (int, error) { if sc.buffer.Len() == 0 { line, err := sc.reader.ReadBytes('\n') if len(line) > 0 { // Convert "2026-11-16" -> "2026-11-16T00:00:00Z" line = invalidDateRegex.ReplaceAll(line, []byte(`:"$1T00:00:00Z"`)) // Convert {publishedDate:""} -> "{publishedDate:null}" line = invalidPublishedDateRegex.ReplaceAll(line, []byte(`"publishedDate":null`)) sc.buffer.Write(line) } if err != nil { if sc.buffer.Len() == 0 { return 0, err } } } return sc.buffer.Read(p) }