package mock import ( "bufio" "bytes" "encoding/json" "errors" "fmt" "log/slog" "math/rand" "os" "path/filepath" "sort" "strconv" "strings" "sync" "time" "git.avsdev.uk/Go/job-search/internal/mock/findatender/ocds" ) // Store keeps every release in memory and appends each new one to a JSON Lines // file, so releases survive restarts. Releases are kept in date order. type Store struct { mu sync.RWMutex file *os.File releases []ocds.Release times []time.Time latest map[string]int // ocid -> index of its most recent release lastNumber map[int]int // year -> highest notice number used // Procedures not yet awarded, as a slice plus positions so that adding, // removing and picking one at random are all constant time. open []string openPosition map[string]int opts options } // OpenStore loads the releases stored at path, creating the file and its // directory if needed, and opens it for appending. Lines that can't be read // are skipped, releases are put back in date order if needed, and a record // left incomplete by a crash is ended so new ones start on a fresh line. // It logs to slog.Default() unless given WithLogger. func OpenStore(path string, opts ...Option) (*Store, error) { store := &Store{ latest: map[string]int{}, lastNumber: map[int]int{}, openPosition: map[string]int{}, opts: applyOptions(opts), } if err := os.MkdirAll(filepath.Dir(path), 0o750); err != nil { return nil, err } releases, err := readReleases(store.opts.logger("store"), path) if err != nil { return nil, err } // The file is written in order, but sort anyway in case it was hand-edited. sort.SliceStable(releases, func(i, other int) bool { return releases[i].Date.Before(releases[other].Date) }) for _, release := range releases { store.releases = append(store.releases, release) store.times = append(store.times, release.Date) } for i, release := range store.releases { store.index(i, release) } if store.file, err = openForAppend(path); err != nil { return nil, err } return store, nil } // readReleases reads the releases stored at path, skipping blank lines and // lines that can't be decoded (which are logged to logger). A missing file // holds no releases. func readReleases(logger *slog.Logger, path string) ([]ocds.Release, error) { input, err := os.Open(filepath.Clean(path)) if os.IsNotExist(err) { return nil, nil } if err != nil { return nil, err } defer input.Close() //nolint:errcheck // read only var releases []ocds.Release scanner := bufio.NewScanner(input) scanner.Buffer(make([]byte, 1<<20), 64<<20) for line := 1; scanner.Scan(); line++ { data := bytes.TrimSpace(scanner.Bytes()) if len(data) == 0 { continue } var release ocds.Release if err := json.Unmarshal(data, &release); err != nil { logger.Warn("skipping unreadable line", "line", line, "error", err) continue } releases = append(releases, release) } if err := scanner.Err(); err != nil { return nil, fmt.Errorf("reading %s: %w", path, err) } return releases, nil } // openForAppend opens path for appending releases. If a previous run died // mid-write, it ends the partial record so the next one starts on a fresh // line. func openForAppend(path string) (*os.File, error) { file, err := os.OpenFile( filepath.Clean(path), os.O_CREATE|os.O_RDWR|os.O_APPEND, 0o600, ) if err != nil { return nil, err } if info, err := file.Stat(); err == nil && info.Size() > 0 { last := make([]byte, 1) _, readErr := file.ReadAt(last, info.Size()-1) if readErr == nil && last[0] != '\n' { if _, err := file.Write([]byte("\n")); err != nil { return nil, errors.Join( fmt.Errorf("repairing %s: %w", path, err), file.Close(), ) } } } return file, nil } // index records release, at position i, in the lookups: the ocid's latest // release, whether its procedure is still open (an award closes it), and the // highest notice number used each year. func (s *Store) index(i int, release ocds.Release) { s.latest[release.OCID] = i if len(release.Tag) > 0 && release.Tag[0] == "award" { s.closeProcedure(release.OCID) } else if _, ok := s.openPosition[release.OCID]; !ok { s.openPosition[release.OCID] = len(s.open) s.open = append(s.open, release.OCID) } if numberText, year, ok := strings.Cut(release.ID, "-"); ok { number, numberErr := strconv.Atoi(numberText) yearNumber, yearErr := strconv.Atoi(year) if numberErr == nil && yearErr == nil && number > s.lastNumber[yearNumber] { s.lastNumber[yearNumber] = number } } } // closeProcedure removes ocid from the open procedures, if it is there. func (s *Store) closeProcedure(ocid string) { position, ok := s.openPosition[ocid] if !ok { return } last := len(s.open) - 1 s.open[position] = s.open[last] s.openPosition[s.open[position]] = position s.open = s.open[:last] delete(s.openPosition, ocid) } // Append persists a release and flushes it to disk. func (s *Store) Append(release ocds.Release) error { return s.append(release, true) } // AppendNoSync persists a release without an fsync; call Sync afterwards. // Used for bulk catch-up, where a sync per record would be very slow. func (s *Store) AppendNoSync(release ocds.Release) error { return s.append(release, false) } // Sync flushes appended releases to disk. Call it after AppendNoSync. func (s *Store) Sync() error { s.mu.Lock() defer s.mu.Unlock() return s.file.Sync() } // append stores release, syncing the file to disk if sync is true. A date // earlier than the newest stored release (for example after a clock change) is // moved up so the order stays valid. func (s *Store) append(release ocds.Release, sync bool) error { s.mu.Lock() defer s.mu.Unlock() release.Date = release.Date.UTC() if count := len(s.times); count > 0 && release.Date.Before(s.times[count-1]) { release.Date = s.times[count-1] } data, err := json.Marshal(release) if err != nil { return err } if _, err := s.file.Write(append(data, '\n')); err != nil { return err } if sync { if err := s.file.Sync(); err != nil { return err } } s.releases = append(s.releases, release) s.times = append(s.times, release.Date) s.index(len(s.releases)-1, release) return nil } // Page returns up to limit releases dated in [from, upper], starting no earlier // than position start, plus the position to continue from and whether more // exist. func (s *Store) Page( from, upper time.Time, start, limit int, ) ([]ocds.Release, int, bool) { s.mu.RLock() defer s.mu.RUnlock() i := sort.Search(len(s.times), func(i int) bool { return !s.times[i].Before(from) }) if start > i { i = start } out := make([]ocds.Release, 0, limit) for i < len(s.releases) && len(out) < limit && !s.times[i].After(upper) { out = append(out, s.releases[i]) i++ } more := i < len(s.releases) && !s.times[i].After(upper) return out, i, more } // Last returns the date of the newest release, or false if the store is // empty. func (s *Store) Last() (time.Time, bool) { s.mu.RLock() defer s.mu.RUnlock() if len(s.times) == 0 { return time.Time{}, false } return s.times[len(s.times)-1], true } // Len returns the number of stored releases. func (s *Store) Len() int { s.mu.RLock() defer s.mu.RUnlock() return len(s.releases) } // HasOCID reports whether any release of the procedure ocid is stored. func (s *Store) HasOCID(ocid string) bool { s.mu.RLock() defer s.mu.RUnlock() _, ok := s.latest[ocid] return ok } // NextNoticeID returns the next FTS-style notice id ("000123-2026") for year. func (s *Store) NextNoticeID(year int) string { s.mu.RLock() defer s.mu.RUnlock() return fmt.Sprintf("%06d-%d", s.lastNumber[year]+1, year) } // RandomOpen picks a procedure that has not been awarded and whose latest // release is after since. Procedures idle since before then are dropped from // the open set, as if they had been abandoned. func (s *Store) RandomOpen( rng *rand.Rand, since time.Time, ) (ocds.Release, bool) { s.mu.Lock() defer s.mu.Unlock() for len(s.open) > 0 { ocid := s.open[rng.Intn(len(s.open))] i := s.latest[ocid] if s.times[i].After(since) { return s.releases[i], true } s.closeProcedure(ocid) } return ocds.Release{}, false } // Close closes the store's file. The store must not be used afterwards. func (s *Store) Close() error { s.mu.Lock() defer s.mu.Unlock() return s.file.Close() }