60 lines
1.9 KiB
Go
60 lines
1.9 KiB
Go
/*
|
|
Copyright © 2026 Craig Williams <craig@avsdev.uk>
|
|
|
|
This program is free software: you can redistribute it and/or modify
|
|
it under the terms of the GNU Lesser General Public License as published by
|
|
the Free Software Foundation, either version 3 of the License, or
|
|
(at your option) any later version.
|
|
|
|
This program is distributed in the hope that it will be useful,
|
|
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
GNU Lesser General Public License for more details.
|
|
|
|
You should have received a copy of the GNU Lesser General Public License
|
|
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
*/
|
|
package findATender
|
|
|
|
import (
|
|
"bufio"
|
|
"bytes"
|
|
"io"
|
|
"regexp"
|
|
)
|
|
|
|
// Global regex to identify and catch date-only fields
|
|
var invalidDateRegex = regexp.MustCompile(`:\s*"(\d{4}-\d{2}-\d{2})"\s*([,}\s])`)
|
|
var invalidPublishedDateRegex = regexp.MustCompile(`"publishedDate":\s*""`)
|
|
|
|
// The UK Government FindATender service doesn't strictly follow OCDS date
|
|
// format. Sometimes it provides dates only, sometimes it sends empty strings
|
|
// instead of nulls.
|
|
type FindATenderStreamCleaner struct {
|
|
reader *bufio.Reader
|
|
buffer bytes.Buffer
|
|
}
|
|
|
|
func NewOCDSStreamCleaner(r io.Reader) *FindATenderStreamCleaner {
|
|
return &FindATenderStreamCleaner{reader: bufio.NewReader(r)}
|
|
}
|
|
|
|
func (sc *FindATenderStreamCleaner) Read(p []byte) (int, error) {
|
|
if sc.buffer.Len() == 0 {
|
|
line, err := sc.reader.ReadBytes('\n')
|
|
if len(line) > 0 {
|
|
// Convert "2026-11-16" -> "2026-11-16T00:00:00Z"
|
|
line = invalidDateRegex.ReplaceAll(line, []byte(`:"$1T00:00:00Z"`))
|
|
// Convert {publishedDate:""} -> "{publishedDate:null}"
|
|
line = invalidPublishedDateRegex.ReplaceAll(line, []byte(`"publishedDate":null`))
|
|
sc.buffer.Write(line)
|
|
}
|
|
if err != nil {
|
|
if sc.buffer.Len() == 0 {
|
|
return 0, err
|
|
}
|
|
}
|
|
}
|
|
return sc.buffer.Read(p)
|
|
}
|