Added fetch service and a mock Find a Tender server
Fetching: - Add `fetch service`, which runs a worker per enabled source until SIGINT/SIGTERM (internal/fetcher) - Add util.Worker: runs a task on an interval with a since-checkpoint that only advances on success, and a shutdown timeout for in-flight work; plus AwaitTask/AwaitTaskE for calls that ignore cancellation - Add fts_mock and sleeper sources - Sleeper: sleeps periodically to emulate work being done. Can be instructed to block the shutdown signal or handle it cleanly Mock FTS server (cmd/mock-fts, internal/mock/findATender): - Serves GET /api/1.0/ocdsReleasePackages with FTS paging, query validation, UK local time handling and per-client rate limiting - Generates fictional releases from an embedded catalog, moving procedures through planning, tender and award, and keeps them in a JSON Lines store that survives restarts - Add logging.AccessLog for combined/common access logs
This commit is contained in:
bwilliams committed
1 parent
dcb714dc19
commit
5cb3601e1f
23 files changed
+97717
-3
No files matched your search
@@ -0,0 +1,184 @@
|
||||
package mock
|
||||
|
||||
import (
|
||||
_ "embed"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"math/rand"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"git.avsdev.uk/Go/job-search/internal/mock/findATender/ocds"
|
||||
)
|
||||
|
||||
// Method is a procurement method and its details, as published on a tender.
|
||||
type Method struct {
|
||||
Method string `json:"procurementMethod"`
|
||||
Details string `json:"procurementMethodDetails,omitempty"`
|
||||
}
|
||||
|
||||
// embeddedCatalog is data/catalog.json, compiled into the binary. Rebuild after
|
||||
// changing the file.
|
||||
//
|
||||
//go:embed data/catalog.json
|
||||
var embeddedCatalog []byte
|
||||
|
||||
// Catalog is the pool the generator draws from. It is read from
|
||||
// ./data/catalog.json. Classifications holds every classification the subjects
|
||||
// use, keyed "{scheme}-{id}" (e.g. "CPV-90513000"), and subjects and their
|
||||
// items refer to entries by that key.
|
||||
type Catalog struct {
|
||||
Buyers []ocds.Organization `json:"buyers"`
|
||||
Suppliers []ocds.Organization `json:"suppliers"`
|
||||
Subjects []Subject `json:"subjects"`
|
||||
Methods []Method `json:"methods"`
|
||||
Classifications map[string]ocds.Classification `json:"classifications"`
|
||||
}
|
||||
|
||||
// Subject is a tender as stored in the catalog: an ocds.Tender whose main
|
||||
// classification and additionalClassifications, on the tender and on each
|
||||
// item, are keys into Catalog.Classifications. The outer fields shadow the
|
||||
// embedded ones of the same JSON name, so everything else is read as
|
||||
// ocds.Tender.
|
||||
type Subject struct {
|
||||
ocds.Tender
|
||||
Classification string `json:"classification,omitempty"`
|
||||
AdditionalClassifications []string `json:"additionalClassifications,omitempty"`
|
||||
Items []SubjectItem `json:"items,omitempty"`
|
||||
}
|
||||
|
||||
// SubjectItem is an ocds.Item whose additionalClassifications are keys into
|
||||
// Catalog.Classifications.
|
||||
type SubjectItem struct {
|
||||
ocds.Item
|
||||
AdditionalClassifications []string `json:"additionalClassifications,omitempty"`
|
||||
}
|
||||
|
||||
// LoadCatalog parses the embedded catalog and checks it is usable: every
|
||||
// pool has at least one entry and every classification key resolves.
|
||||
func LoadCatalog() (*Catalog, error) {
|
||||
return parseCatalog(embeddedCatalog, "embedded catalog")
|
||||
}
|
||||
|
||||
// parseCatalog decodes and validates catalog JSON; name labels any error.
|
||||
func parseCatalog(b []byte, name string) (*Catalog, error) {
|
||||
var c Catalog
|
||||
if err := json.Unmarshal(b, &c); err != nil {
|
||||
return nil, fmt.Errorf("decoding %s: %w", name, err)
|
||||
}
|
||||
if err := c.validate(); err != nil {
|
||||
return nil, fmt.Errorf("invalid %s: %w", name, err)
|
||||
}
|
||||
return &c, nil
|
||||
}
|
||||
|
||||
func (c *Catalog) validate() error {
|
||||
var errs []error
|
||||
|
||||
for _, pool := range []struct {
|
||||
name string
|
||||
n int
|
||||
}{
|
||||
{"buyers", len(c.Buyers)},
|
||||
{"suppliers", len(c.Suppliers)},
|
||||
{"subjects", len(c.Subjects)},
|
||||
{"methods", len(c.Methods)},
|
||||
{"classifications", len(c.Classifications)},
|
||||
} {
|
||||
if pool.n == 0 {
|
||||
errs = append(errs, fmt.Errorf("no %s", pool.name))
|
||||
}
|
||||
}
|
||||
|
||||
missing := map[string]bool{}
|
||||
check := func(keys ...string) {
|
||||
for _, k := range keys {
|
||||
if _, ok := c.Classifications[k]; k != "" && !ok {
|
||||
missing[k] = true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for _, s := range c.Subjects {
|
||||
check(s.Classification)
|
||||
check(s.AdditionalClassifications...)
|
||||
for _, it := range s.Items {
|
||||
check(it.AdditionalClassifications...)
|
||||
}
|
||||
}
|
||||
|
||||
if len(missing) > 0 {
|
||||
keys := make([]string, 0, len(missing))
|
||||
for k := range missing {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
if len(keys) > 5 {
|
||||
keys = append(keys[:5], "...")
|
||||
}
|
||||
errs = append(errs, fmt.Errorf(
|
||||
"%d classification key(s) not found in classifications: %s",
|
||||
len(missing), strings.Join(keys, ", "),
|
||||
))
|
||||
}
|
||||
|
||||
return errors.Join(errs...)
|
||||
}
|
||||
|
||||
// ---------- picking (always returns a copy the caller may modify) ----------
|
||||
|
||||
func (c *Catalog) buyer(r *rand.Rand) ocds.Organization {
|
||||
return clone(c.Buyers[r.Intn(len(c.Buyers))])
|
||||
}
|
||||
|
||||
func (c *Catalog) supplier(r *rand.Rand) ocds.Organization {
|
||||
return clone(c.Suppliers[r.Intn(len(c.Suppliers))])
|
||||
}
|
||||
|
||||
func (c *Catalog) method(r *rand.Rand) Method {
|
||||
return c.Methods[r.Intn(len(c.Methods))]
|
||||
}
|
||||
|
||||
// subject picks a subject and returns it as an ocds.Tender, with its
|
||||
// classification keys resolved to full classifications.
|
||||
func (c *Catalog) subject(r *rand.Rand) ocds.Tender {
|
||||
s := clone(c.Subjects[r.Intn(len(c.Subjects))])
|
||||
t := s.Tender
|
||||
t.Classification = c.resolveOne(s.Classification)
|
||||
t.AdditionalClassifications = c.resolve(s.AdditionalClassifications)
|
||||
t.Items = nil
|
||||
for _, it := range s.Items {
|
||||
item := it.Item
|
||||
item.AdditionalClassifications = c.resolve(it.AdditionalClassifications)
|
||||
t.Items = append(t.Items, item)
|
||||
}
|
||||
return t
|
||||
}
|
||||
|
||||
func (c *Catalog) resolveOne(key string) *ocds.Classification {
|
||||
if cl, ok := c.Classifications[key]; ok {
|
||||
return &cl
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Catalog) resolve(keys []string) []ocds.Classification {
|
||||
var out []ocds.Classification
|
||||
for _, k := range keys {
|
||||
if cl, ok := c.Classifications[k]; ok {
|
||||
out = append(out, cl)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// clone deep-copies a value through JSON, which is all the generated types need.
|
||||
func clone[T any](v T) T {
|
||||
var out T
|
||||
b, err := json.Marshal(v)
|
||||
if err == nil {
|
||||
_ = json.Unmarshal(b, &out)
|
||||
}
|
||||
return out
|
||||
}
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,13 @@
|
||||
// Package mock implements a mock of the Find a Tender (FTS) OCDS release
|
||||
// package endpoint (GET /api/1.0/ocdsReleasePackages): a catalog of buyers,
|
||||
// suppliers, subjects and methods loaded from catalog.json, a persistent
|
||||
// release store, a generator with a background worker, the HTTP handler and a
|
||||
// per-client rate limiter.
|
||||
package mock
|
||||
|
||||
// Files kept in the mock-data directory.
|
||||
//
|
||||
// ReleasesFile is created and appended to by the mock.
|
||||
const (
|
||||
ReleasesFile = "releases.jsonl"
|
||||
)
|
||||
@@ -0,0 +1,274 @@
|
||||
package mock
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"hash/fnv"
|
||||
"log"
|
||||
"math"
|
||||
"math/rand"
|
||||
"time"
|
||||
|
||||
"git.avsdev.uk/Go/job-search/internal/mock/findATender/ocds"
|
||||
)
|
||||
|
||||
const ocidPrefix = "ocds-mock-"
|
||||
|
||||
// Generator makes new releases from the catalog. About a third of the time it
|
||||
// moves an existing procedure on (planning -> tender -> award) so the same ocid
|
||||
// appears in several releases, as it does on the real service.
|
||||
type Generator struct {
|
||||
cat *Catalog
|
||||
store *Store
|
||||
rnd *rand.Rand
|
||||
}
|
||||
|
||||
func NewGenerator(cat *Catalog, store *Store, seed int64) *Generator {
|
||||
return &Generator{
|
||||
cat: cat,
|
||||
store: store,
|
||||
rnd: rand.New(rand.NewSource(seed)),
|
||||
}
|
||||
}
|
||||
|
||||
func (g *Generator) Make(at time.Time) ocds.Release {
|
||||
at = at.UTC().Truncate(time.Second)
|
||||
if g.rnd.Float64() < 0.35 {
|
||||
if prev, ok := g.store.RandomOpen(g.rnd, at.AddDate(0, 0, -60)); ok {
|
||||
return g.advance(prev, at)
|
||||
}
|
||||
}
|
||||
return g.newProcedure(at)
|
||||
}
|
||||
|
||||
func (g *Generator) newProcedure(at time.Time) ocds.Release {
|
||||
buyer := g.cat.buyer(g.rnd)
|
||||
ensurePartyID(&buyer)
|
||||
buyer.Roles = []string{"buyer"}
|
||||
m := g.cat.method(g.rnd)
|
||||
|
||||
tag, status := "tender", "active"
|
||||
if g.rnd.Float64() < 0.2 {
|
||||
tag, status = "planning", "planned"
|
||||
}
|
||||
ocid := g.newOCID()
|
||||
|
||||
t := g.cat.subject(g.rnd)
|
||||
t.ID = ocds.ID(ocid)
|
||||
t.ProcurementMethod = m.Method
|
||||
t.ProcurementMethodDetails = m.Details
|
||||
|
||||
factor := 0.5 + g.rnd.Float64()*1.5
|
||||
if t.Value == nil {
|
||||
amount := float64((g.rnd.Intn(5000) + 10) * 1000)
|
||||
t.Value = &ocds.Value{
|
||||
Amount: &amount,
|
||||
Currency: "GBP",
|
||||
}
|
||||
} else {
|
||||
scaleValue(t.Value, factor)
|
||||
}
|
||||
|
||||
for i := range t.Lots {
|
||||
scaleValue(t.Lots[i].Value, factor)
|
||||
}
|
||||
|
||||
if len(t.Lots) == 0 {
|
||||
v := clone(*t.Value)
|
||||
t.Lots = []ocds.Lot{{
|
||||
ID: "1",
|
||||
Title: t.Title,
|
||||
Value: &v,
|
||||
}}
|
||||
}
|
||||
|
||||
setStatus(&t, status)
|
||||
if status == "active" {
|
||||
t.TenderPeriod = &ocds.Period{
|
||||
EndDate: ptr(at.AddDate(0, 0, 30+g.rnd.Intn(30))),
|
||||
}
|
||||
}
|
||||
|
||||
return ocds.Release{
|
||||
OCID: ocid,
|
||||
ID: g.store.NextNoticeID(at.Year()),
|
||||
Date: at,
|
||||
Tag: []string{tag},
|
||||
InitiationType: "tender",
|
||||
Parties: []ocds.Organization{buyer},
|
||||
Buyer: &ocds.OrganizationReference{
|
||||
ID: ocds.ID(buyer.ID),
|
||||
Name: buyer.Name,
|
||||
},
|
||||
Tender: &t,
|
||||
Language: "en",
|
||||
}
|
||||
}
|
||||
|
||||
func (g *Generator) advance(prev ocds.Release, at time.Time) ocds.Release {
|
||||
r := clone(prev)
|
||||
r.ID = g.store.NextNoticeID(at.Year())
|
||||
r.Date = at
|
||||
r.Awards = nil
|
||||
if r.Tender == nil {
|
||||
r.Tender = &ocds.Tender{
|
||||
ID: ocds.ID(r.OCID),
|
||||
}
|
||||
}
|
||||
|
||||
next := "award"
|
||||
switch prev.Tag[0] {
|
||||
case "planning":
|
||||
next = "tender"
|
||||
case "tender":
|
||||
if g.rnd.Float64() < 0.25 {
|
||||
next = "tenderUpdate"
|
||||
}
|
||||
}
|
||||
r.Tag = []string{next}
|
||||
|
||||
switch next {
|
||||
case "tender":
|
||||
setStatus(r.Tender, "active")
|
||||
r.Tender.TenderPeriod = &ocds.Period{
|
||||
EndDate: ptr(at.AddDate(0, 0, 30+g.rnd.Intn(30))),
|
||||
}
|
||||
case "tenderUpdate":
|
||||
// A typical update: the deadline is extended by two weeks.
|
||||
end := at.AddDate(0, 0, 30)
|
||||
if tp := r.Tender.TenderPeriod; tp != nil && tp.EndDate != nil {
|
||||
end = *tp.EndDate
|
||||
}
|
||||
r.Tender.TenderPeriod = &ocds.Period{
|
||||
EndDate: ptr(end.AddDate(0, 0, 14)),
|
||||
}
|
||||
case "award":
|
||||
setStatus(r.Tender, "complete")
|
||||
sup := g.cat.supplier(g.rnd)
|
||||
ensurePartyID(&sup)
|
||||
sup.Roles = []string{"supplier"}
|
||||
r.Parties = append(r.Parties, sup)
|
||||
|
||||
var value *ocds.Value
|
||||
if r.Tender.Value != nil {
|
||||
v := clone(*r.Tender.Value)
|
||||
scaleValue(&v, 0.8+g.rnd.Float64()*0.2)
|
||||
value = &v
|
||||
}
|
||||
|
||||
var lotIDs []string
|
||||
for _, lot := range r.Tender.Lots {
|
||||
if lot.ID != "" {
|
||||
lotIDs = append(lotIDs, lot.ID)
|
||||
}
|
||||
}
|
||||
|
||||
r.Awards = []ocds.Award{{
|
||||
ID: "1",
|
||||
Status: "active",
|
||||
Date: ptr(at),
|
||||
Value: value,
|
||||
Suppliers: []ocds.OrganizationReference{
|
||||
{
|
||||
ID: ocds.ID(sup.ID),
|
||||
Name: sup.Name,
|
||||
},
|
||||
},
|
||||
RelatedLots: lotIDs,
|
||||
}}
|
||||
}
|
||||
|
||||
return r
|
||||
}
|
||||
|
||||
func (g *Generator) newOCID() string {
|
||||
for {
|
||||
id := fmt.Sprintf("%s%06x", ocidPrefix, g.rnd.Intn(1<<24))
|
||||
if !g.store.HasOCID(id) {
|
||||
return id
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (g *Generator) gap(min, max time.Duration) time.Duration {
|
||||
if max <= min {
|
||||
return min
|
||||
}
|
||||
return min + time.Duration(g.rnd.Int63n(int64(max-min)+1))
|
||||
}
|
||||
|
||||
// CatchUp fills the period since the last stored release (or the last backfill
|
||||
// period, whichever is shorter) so restarts leave no gap in the timeline.
|
||||
func (g *Generator) CatchUp(
|
||||
now time.Time, backfill, min, max time.Duration,
|
||||
) (int, error) {
|
||||
start := now.Add(-backfill)
|
||||
if last, ok := g.store.Last(); ok && last.After(start) {
|
||||
start = last
|
||||
}
|
||||
|
||||
n := 0
|
||||
for t := start.Add(g.gap(min, max)); !t.After(now); t = t.Add(g.gap(min, max)) {
|
||||
if err := g.store.AppendNoSync(g.Make(t)); err != nil {
|
||||
return n, err
|
||||
}
|
||||
n++
|
||||
}
|
||||
|
||||
return n, g.store.Sync()
|
||||
}
|
||||
|
||||
// Run publishes a new release after a random gap between min and max until ctx ends.
|
||||
func (g *Generator) Run(ctx context.Context, min, max time.Duration) {
|
||||
for {
|
||||
d := g.gap(min, max)
|
||||
log.Printf("generator: next release in %s", d.Round(time.Second))
|
||||
|
||||
timer := time.NewTimer(d)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
timer.Stop()
|
||||
return
|
||||
case <-timer.C:
|
||||
}
|
||||
|
||||
r := g.Make(time.Now())
|
||||
if err := g.store.Append(r); err != nil {
|
||||
log.Printf("generator: storing release: %v", err)
|
||||
continue
|
||||
}
|
||||
|
||||
log.Printf("generator: published %s (%s) %s", r.ID, r.Tag[0], r.OCID)
|
||||
}
|
||||
}
|
||||
|
||||
// ---------- helpers ----------
|
||||
|
||||
func ptr[T any](v T) *T { return &v }
|
||||
|
||||
func setStatus(t *ocds.Tender, status string) {
|
||||
t.Status = status
|
||||
for i := range t.Lots {
|
||||
t.Lots[i].Status = status
|
||||
}
|
||||
}
|
||||
|
||||
func scaleValue(v *ocds.Value, f float64) {
|
||||
if v == nil {
|
||||
return
|
||||
}
|
||||
for _, a := range []**float64{&v.Amount, &v.AmountGross} {
|
||||
if *a != nil {
|
||||
*a = ptr(math.Round(**a*f/100) * 100)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func ensurePartyID(p *ocds.Organization) {
|
||||
if p.ID != "" {
|
||||
return
|
||||
}
|
||||
h := fnv.New32a()
|
||||
h.Write([]byte(p.Name))
|
||||
p.ID = fmt.Sprintf("GB-FTS-%d", h.Sum32()%1000000)
|
||||
}
|
||||
@@ -0,0 +1,189 @@
|
||||
package mock
|
||||
|
||||
import (
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.avsdev.uk/Go/job-search/internal/mock/findATender/ocds"
|
||||
)
|
||||
|
||||
const (
|
||||
defaultLimit = 100
|
||||
maxLimit = 100
|
||||
timeLayout = "2006-01-02T15:04:05"
|
||||
BasePath = "/api/1.0/ocdsReleasePackages"
|
||||
)
|
||||
|
||||
type server struct {
|
||||
store *Store
|
||||
baseURL string
|
||||
loc *time.Location // zone for updatedFrom/updatedTo without an offset; nil means UTC
|
||||
}
|
||||
|
||||
// NewHandler serves release packages from store. baseURL is the public base
|
||||
// URL used in each package's uri and links.next. loc is the time zone that
|
||||
// updatedFrom and updatedTo are read in when they carry no offset, as the
|
||||
// documented yyyy-MM-ddTHH:mm:ss format doesn't (Europe/London for FTS).
|
||||
// Stored release dates are UTC; comparisons are on absolute instants.
|
||||
func NewHandler(
|
||||
store *Store, baseURL string, loc *time.Location,
|
||||
) http.HandlerFunc {
|
||||
return (&server{store: store, baseURL: baseURL, loc: loc}).handle
|
||||
}
|
||||
|
||||
func encodeCursor(i int) string {
|
||||
return base64.RawURLEncoding.EncodeToString([]byte("idx:" + strconv.Itoa(i)))
|
||||
}
|
||||
|
||||
func decodeCursor(c string) (int, error) {
|
||||
b, err := base64.RawURLEncoding.DecodeString(c)
|
||||
if err != nil || !strings.HasPrefix(string(b), "idx:") {
|
||||
return 0, fmt.Errorf("invalid cursor")
|
||||
}
|
||||
|
||||
n, err := strconv.Atoi(strings.TrimPrefix(string(b), "idx:"))
|
||||
if err != nil || n < 0 {
|
||||
return 0, fmt.Errorf("invalid cursor")
|
||||
}
|
||||
|
||||
return n, nil
|
||||
}
|
||||
|
||||
func badRequest(w http.ResponseWriter, msg string) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(http.StatusBadRequest)
|
||||
json.NewEncoder(w).Encode(map[string]string{"error": msg})
|
||||
}
|
||||
|
||||
// parseTime reads a query timestamp. Values without an offset (the documented
|
||||
// yyyy-MM-ddTHH:mm:ss, or a plain date) are local time in loc; values with an
|
||||
// offset or Z (RFC 3339) mean exactly what they say.
|
||||
func parseTime(v string, loc *time.Location) (time.Time, error) {
|
||||
if loc == nil {
|
||||
loc = time.UTC
|
||||
}
|
||||
|
||||
if t, err := time.Parse(time.RFC3339, v); err == nil {
|
||||
return t, nil
|
||||
}
|
||||
|
||||
for _, layout := range []string{timeLayout, "2006-01-02"} {
|
||||
if t, err := time.ParseInLocation(layout, v, loc); err == nil {
|
||||
return t, nil
|
||||
}
|
||||
}
|
||||
|
||||
return time.Time{}, fmt.Errorf("must be in format yyyy-MM-ddTHH:mm:ss")
|
||||
}
|
||||
|
||||
// pageQuery is a parsed release package request.
|
||||
type pageQuery struct {
|
||||
from, upper time.Time // releases dated in [from, upper]
|
||||
start int // index of the first release to return
|
||||
limit int
|
||||
}
|
||||
|
||||
func (s *server) handle(w http.ResponseWriter, req *http.Request) {
|
||||
if req.Method != http.MethodGet {
|
||||
w.WriteHeader(http.StatusMethodNotAllowed)
|
||||
return
|
||||
}
|
||||
|
||||
q := req.URL.Query()
|
||||
pq, err := s.parseQuery(q)
|
||||
if err != nil {
|
||||
badRequest(w, err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
releases, next, more := s.store.Page(pq.from, pq.upper, pq.start, pq.limit)
|
||||
pkg := ocds.ReleasePackage{
|
||||
URI: s.baseURL + req.URL.RequestURI(),
|
||||
Version: "1.1",
|
||||
Extensions: ocds.DeclaredExtensions,
|
||||
PublishedDate: ptr(time.Now().UTC().Truncate(time.Second)),
|
||||
Publisher: &ocds.ReleasePackagePublisher{
|
||||
Name: "OCDS Mock",
|
||||
URI: s.baseURL,
|
||||
},
|
||||
Releases: releases,
|
||||
}
|
||||
if more {
|
||||
pkg.Links = &ocds.ReleasePackageLinks{Next: s.nextLink(q, next)}
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "application/json; charset=utf-8")
|
||||
w.Header().Set("Access-Control-Allow-Origin", "*")
|
||||
enc := json.NewEncoder(w)
|
||||
enc.SetIndent("", " ")
|
||||
enc.SetEscapeHTML(false)
|
||||
if err := enc.Encode(pkg); err != nil {
|
||||
log.Printf("handler: writing response: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// parseQuery reads and checks the query parameters. Its errors are meant for
|
||||
// the client.
|
||||
func (s *server) parseQuery(q url.Values) (pageQuery, error) {
|
||||
pq := pageQuery{limit: defaultLimit, upper: time.Now().UTC()}
|
||||
|
||||
if v := q.Get("limit"); v != "" {
|
||||
n, err := strconv.Atoi(v)
|
||||
if err != nil || n < 1 || n > maxLimit {
|
||||
return pq, fmt.Errorf("limit must be an integer between 1 and %d", maxLimit)
|
||||
}
|
||||
pq.limit = n
|
||||
}
|
||||
|
||||
if v := q.Get("updatedFrom"); v != "" {
|
||||
t, err := parseTime(v, s.loc)
|
||||
if err != nil {
|
||||
return pq, fmt.Errorf("updatedFrom %w", err)
|
||||
}
|
||||
pq.from = t
|
||||
}
|
||||
|
||||
if v := q.Get("updatedTo"); v != "" {
|
||||
t, err := parseTime(v, s.loc)
|
||||
if err != nil {
|
||||
return pq, fmt.Errorf("updatedTo %w", err)
|
||||
}
|
||||
if q.Get("updatedFrom") != "" && t.Before(pq.from) {
|
||||
return pq, errors.New("updatedTo must not be before updatedFrom")
|
||||
}
|
||||
if t.Before(pq.upper) {
|
||||
pq.upper = t
|
||||
}
|
||||
}
|
||||
|
||||
if c := q.Get("cursor"); c != "" {
|
||||
n, err := decodeCursor(c)
|
||||
if err != nil {
|
||||
return pq, err
|
||||
}
|
||||
pq.start = n
|
||||
}
|
||||
|
||||
return pq, nil
|
||||
}
|
||||
|
||||
// nextLink is the URL of the page starting at release index next: the same
|
||||
// query with a new cursor.
|
||||
func (s *server) nextLink(q url.Values, next int) string {
|
||||
nq := url.Values{}
|
||||
for k, vs := range q {
|
||||
if k != "cursor" {
|
||||
nq[k] = vs
|
||||
}
|
||||
}
|
||||
nq.Set("cursor", encodeCursor(next))
|
||||
return s.baseURL + BasePath + "?" + nq.Encode()
|
||||
}
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,119 @@
|
||||
package mock
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// ---------- rate limiting ----------
|
||||
|
||||
type clientWindow struct {
|
||||
hour time.Time // clock hour the anchor belongs to
|
||||
anchor time.Time // first request of that hour
|
||||
windowStart time.Time // start of the current window
|
||||
count int // accepted requests in the current window
|
||||
}
|
||||
|
||||
type RateLimiter struct {
|
||||
mu sync.Mutex
|
||||
limit int
|
||||
window time.Duration
|
||||
clients map[string]*clientWindow
|
||||
now func() time.Time
|
||||
}
|
||||
|
||||
func NewRateLimiter(limit int, window time.Duration) *RateLimiter {
|
||||
return &RateLimiter{
|
||||
limit: limit,
|
||||
window: window,
|
||||
clients: map[string]*clientWindow{},
|
||||
now: time.Now,
|
||||
}
|
||||
}
|
||||
|
||||
// allow records a request for key and reports whether it may proceed. When it
|
||||
// may not, it also returns the whole seconds until the client's next window.
|
||||
func (rl *RateLimiter) allow(key string) (bool, int) {
|
||||
rl.mu.Lock()
|
||||
defer rl.mu.Unlock()
|
||||
|
||||
now := rl.now().UTC()
|
||||
hour := now.Truncate(time.Hour)
|
||||
|
||||
c, ok := rl.clients[key]
|
||||
if !ok || !c.hour.Equal(hour) {
|
||||
// First request of this clock hour: it anchors the windows.
|
||||
if !ok && len(rl.clients) > 10000 {
|
||||
rl.prune(hour)
|
||||
}
|
||||
c = &clientWindow{hour: hour, anchor: now, windowStart: now}
|
||||
rl.clients[key] = c
|
||||
}
|
||||
|
||||
// Move to whichever fixed window, counted from the anchor, now falls in.
|
||||
idx := now.Sub(c.anchor) / rl.window
|
||||
if ws := c.anchor.Add(idx * rl.window); !ws.Equal(c.windowStart) {
|
||||
c.windowStart = ws
|
||||
c.count = 0
|
||||
}
|
||||
|
||||
if c.count < rl.limit {
|
||||
c.count++
|
||||
return true, 0
|
||||
}
|
||||
|
||||
// The next window opens at the end of this one, or at the top of the
|
||||
// next hour if that comes first (because a new hour re-anchors).
|
||||
next := c.windowStart.Add(rl.window)
|
||||
if nh := hour.Add(time.Hour); nh.Before(next) {
|
||||
next = nh
|
||||
}
|
||||
secs := int((next.Sub(now) + time.Second - 1) / time.Second) // round up
|
||||
if secs < 1 {
|
||||
secs = 1
|
||||
}
|
||||
return false, secs
|
||||
}
|
||||
|
||||
// prune drops clients whose state belongs to an earlier hour.
|
||||
func (rl *RateLimiter) prune(hour time.Time) {
|
||||
for k, c := range rl.clients {
|
||||
if c.hour.Before(hour) {
|
||||
delete(rl.clients, k)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// clientKey produces an identifier for the client based on remote address.
|
||||
// Could be updated to support proxy and forwarded-for headers.
|
||||
func clientKey(req *http.Request) string {
|
||||
host, _, err := net.SplitHostPort(req.RemoteAddr)
|
||||
if err != nil {
|
||||
return req.RemoteAddr
|
||||
}
|
||||
return host
|
||||
}
|
||||
|
||||
// Middleware attaches to http handler. Checks to see if a client is allowed
|
||||
// to make a request in their current window or returns a "TooManyRequests"
|
||||
// status and the time the client should wait before making another request.
|
||||
func (rl *RateLimiter) Middleware(next http.HandlerFunc) http.HandlerFunc {
|
||||
return func(w http.ResponseWriter, req *http.Request) {
|
||||
ok, retry := rl.allow(clientKey(req))
|
||||
if !ok {
|
||||
w.Header().Set("Retry-After", strconv.Itoa(retry))
|
||||
w.Header().Set("Content-Type", "text/plain; charset=utf-8")
|
||||
w.WriteHeader(http.StatusTooManyRequests)
|
||||
fmt.Fprintf(
|
||||
w, "Rate limit of %d exceeded. Please retry after %d seconds.",
|
||||
rl.limit, retry,
|
||||
)
|
||||
return
|
||||
}
|
||||
next(w, req)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,257 @@
|
||||
package mock
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log"
|
||||
"math/rand"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"git.avsdev.uk/Go/job-search/internal/mock/findATender/ocds"
|
||||
)
|
||||
|
||||
// Store keeps every release in memory and appends each new one to a JSON Lines
|
||||
// file, so releases survive restarts. Releases are kept in date order.
|
||||
type Store struct {
|
||||
mu sync.RWMutex
|
||||
f *os.File
|
||||
releases []ocds.Release
|
||||
times []time.Time
|
||||
latest map[string]int // ocid -> index of its most recent release
|
||||
lastNum map[int]int // year -> highest notice number used
|
||||
|
||||
// Procedures not yet awarded, as a slice plus positions so that adding,
|
||||
// removing and picking one at random are all constant time.
|
||||
open []string
|
||||
openPos map[string]int
|
||||
}
|
||||
|
||||
func OpenStore(path string) (*Store, error) {
|
||||
s := &Store{latest: map[string]int{}, lastNum: map[int]int{}, openPos: map[string]int{}}
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o750); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
releases, err := readReleases(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// The file is written in order, but sort anyway in case it was hand-edited.
|
||||
sort.SliceStable(releases, func(i, j int) bool {
|
||||
return releases[i].Date.Before(releases[j].Date)
|
||||
})
|
||||
for _, r := range releases {
|
||||
s.releases = append(s.releases, r)
|
||||
s.times = append(s.times, r.Date)
|
||||
}
|
||||
for i, r := range s.releases {
|
||||
s.index(i, r)
|
||||
}
|
||||
|
||||
if s.f, err = openForAppend(path); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return s, nil
|
||||
}
|
||||
|
||||
// readReleases reads the releases stored at path, skipping lines that are
|
||||
// blank or can't be decoded. A missing file holds no releases.
|
||||
func readReleases(path string) ([]ocds.Release, error) {
|
||||
in, err := os.Open(path)
|
||||
if os.IsNotExist(err) {
|
||||
return nil, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer in.Close()
|
||||
|
||||
var releases []ocds.Release
|
||||
sc := bufio.NewScanner(in)
|
||||
sc.Buffer(make([]byte, 1<<20), 64<<20)
|
||||
for line := 1; sc.Scan(); line++ {
|
||||
b := bytes.TrimSpace(sc.Bytes())
|
||||
if len(b) == 0 {
|
||||
continue
|
||||
}
|
||||
var r ocds.Release
|
||||
if err := json.Unmarshal(b, &r); err != nil {
|
||||
log.Printf("store: skipping unreadable line %d: %v", line, err)
|
||||
continue
|
||||
}
|
||||
releases = append(releases, r)
|
||||
}
|
||||
if err := sc.Err(); err != nil {
|
||||
return nil, fmt.Errorf("reading %s: %w", path, err)
|
||||
}
|
||||
return releases, nil
|
||||
}
|
||||
|
||||
// openForAppend opens path for appending releases. If a previous run died
|
||||
// mid-write, it ends the partial record so the next one starts on a fresh
|
||||
// line.
|
||||
func openForAppend(path string) (*os.File, error) {
|
||||
f, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR|os.O_APPEND, 0o600)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if st, err := f.Stat(); err == nil && st.Size() > 0 {
|
||||
last := make([]byte, 1)
|
||||
if _, err := f.ReadAt(last, st.Size()-1); err == nil && last[0] != '\n' {
|
||||
if _, err := f.Write([]byte("\n")); err != nil {
|
||||
f.Close()
|
||||
return nil, fmt.Errorf("repairing %s: %w", path, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
return f, nil
|
||||
}
|
||||
|
||||
func (s *Store) index(i int, r ocds.Release) {
|
||||
s.latest[r.OCID] = i
|
||||
if len(r.Tag) > 0 && r.Tag[0] == "award" {
|
||||
s.closeProcedure(r.OCID)
|
||||
} else if _, ok := s.openPos[r.OCID]; !ok {
|
||||
s.openPos[r.OCID] = len(s.open)
|
||||
s.open = append(s.open, r.OCID)
|
||||
}
|
||||
if num, year, ok := strings.Cut(r.ID, "-"); ok {
|
||||
n, err1 := strconv.Atoi(num)
|
||||
y, err2 := strconv.Atoi(year)
|
||||
if err1 == nil && err2 == nil && n > s.lastNum[y] {
|
||||
s.lastNum[y] = n
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Store) closeProcedure(ocid string) {
|
||||
pos, ok := s.openPos[ocid]
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
last := len(s.open) - 1
|
||||
s.open[pos] = s.open[last]
|
||||
s.openPos[s.open[pos]] = pos
|
||||
s.open = s.open[:last]
|
||||
delete(s.openPos, ocid)
|
||||
}
|
||||
|
||||
// Append persists a release and flushes it to disk.
|
||||
func (s *Store) Append(r ocds.Release) error { return s.append(r, true) }
|
||||
|
||||
// AppendNoSync persists a release without an fsync; call Sync afterwards.
|
||||
// Used for bulk catch-up, where a sync per record would be very slow.
|
||||
func (s *Store) AppendNoSync(r ocds.Release) error { return s.append(r, false) }
|
||||
|
||||
func (s *Store) Sync() error {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
return s.f.Sync()
|
||||
}
|
||||
|
||||
// A date earlier than the newest stored release (for example after a clock
|
||||
// change) is moved up so the order stays valid.
|
||||
func (s *Store) append(r ocds.Release, sync bool) error {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
|
||||
r.Date = r.Date.UTC()
|
||||
if n := len(s.times); n > 0 && r.Date.Before(s.times[n-1]) {
|
||||
r.Date = s.times[n-1]
|
||||
}
|
||||
b, err := json.Marshal(r)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := s.f.Write(append(b, '\n')); err != nil {
|
||||
return err
|
||||
}
|
||||
if sync {
|
||||
if err := s.f.Sync(); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
s.releases = append(s.releases, r)
|
||||
s.times = append(s.times, r.Date)
|
||||
s.index(len(s.releases)-1, r)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Page returns up to limit releases dated in [from, upper], starting no earlier
|
||||
// than position start, plus the position to continue from and whether more exist.
|
||||
func (s *Store) Page(from, upper time.Time, start, limit int) ([]ocds.Release, int, bool) {
|
||||
s.mu.RLock()
|
||||
defer s.mu.RUnlock()
|
||||
|
||||
i := sort.Search(len(s.times), func(i int) bool { return !s.times[i].Before(from) })
|
||||
if start > i {
|
||||
i = start
|
||||
}
|
||||
out := make([]ocds.Release, 0, limit)
|
||||
for i < len(s.releases) && len(out) < limit && !s.times[i].After(upper) {
|
||||
out = append(out, s.releases[i])
|
||||
i++
|
||||
}
|
||||
more := i < len(s.releases) && !s.times[i].After(upper)
|
||||
return out, i, more
|
||||
}
|
||||
|
||||
func (s *Store) Last() (time.Time, bool) {
|
||||
s.mu.RLock()
|
||||
defer s.mu.RUnlock()
|
||||
if len(s.times) == 0 {
|
||||
return time.Time{}, false
|
||||
}
|
||||
return s.times[len(s.times)-1], true
|
||||
}
|
||||
|
||||
func (s *Store) Len() int {
|
||||
s.mu.RLock()
|
||||
defer s.mu.RUnlock()
|
||||
return len(s.releases)
|
||||
}
|
||||
|
||||
func (s *Store) HasOCID(ocid string) bool {
|
||||
s.mu.RLock()
|
||||
defer s.mu.RUnlock()
|
||||
_, ok := s.latest[ocid]
|
||||
return ok
|
||||
}
|
||||
|
||||
// NextNoticeID returns the next FTS-style notice id ("000123-2026") for year.
|
||||
func (s *Store) NextNoticeID(year int) string {
|
||||
s.mu.RLock()
|
||||
defer s.mu.RUnlock()
|
||||
return fmt.Sprintf("%06d-%d", s.lastNum[year]+1, year)
|
||||
}
|
||||
|
||||
// RandomOpen picks a procedure that has not been awarded and whose latest
|
||||
// release is after since. Procedures idle since before then are dropped from
|
||||
// the open set, as if they had been abandoned.
|
||||
func (s *Store) RandomOpen(r *rand.Rand, since time.Time) (ocds.Release, bool) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
for len(s.open) > 0 {
|
||||
ocid := s.open[r.Intn(len(s.open))]
|
||||
i := s.latest[ocid]
|
||||
if s.times[i].After(since) {
|
||||
return s.releases[i], true
|
||||
}
|
||||
s.closeProcedure(ocid)
|
||||
}
|
||||
return ocds.Release{}, false
|
||||
}
|
||||
|
||||
func (s *Store) Close() error {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
return s.f.Close()
|
||||
}
|
||||
+38
@@ -0,0 +1,38 @@
|
||||
{
|
||||
"buyers": [
|
||||
{
|
||||
"id": "ZZ-MPON-PAAA-1111-AAAA",
|
||||
"name": "Example Borough Council",
|
||||
"identifier": {"scheme": "ZZ-MPON", "id": "PAAA-1111-AAAA"},
|
||||
"address": {"locality": "Exampleton", "postalCode": "ZZ1 1AA", "countryName": "United Kingdom", "region": "UKJ"},
|
||||
"details": {"classifications": [{"scheme": "UK_CA_TYPE", "id": "publicAuthoritySubCentralGovernment", "description": "Public authority - sub-central government"}]}
|
||||
},
|
||||
{"id": "ZZ-MPON-PBBB-2222-BBBB", "name": "Example NHS Trust"}
|
||||
],
|
||||
"suppliers": [
|
||||
{"id": "ZZ-MCOH-01234567", "name": "Example Supplies Ltd", "identifier": {"scheme": "ZZ-MCOH", "id": "01234567"}}
|
||||
],
|
||||
"subjects": [
|
||||
{
|
||||
"title": "Grounds maintenance services",
|
||||
"mainProcurementCategory": "services",
|
||||
"classification": "CPV-77314000",
|
||||
"items": [{"id": "1", "additionalClassifications": ["CPV-77310000"], "deliveryAddresses": [{"region": "UKJ"}], "relatedLot": "1"}],
|
||||
"lots": [{"id": "1", "title": "Lot 1: Parks", "value": {"amountGross": 120000, "amount": 100000, "currency": "GBP"}}],
|
||||
"value": {"amountGross": 120000, "amount": 100000, "currency": "GBP"}
|
||||
},
|
||||
{
|
||||
"title": "Patient transport",
|
||||
"classification": "CPV-60130000"
|
||||
}
|
||||
],
|
||||
"methods": [
|
||||
{"procurementMethod": "open", "procurementMethodDetails": "Open procedure"},
|
||||
{"procurementMethod": "selective", "procurementMethodDetails": "Competitive flexible procedure"}
|
||||
],
|
||||
"classifications": {
|
||||
"CPV-60130000": {"scheme": "CPV", "id": "60130000", "description": "Special-purpose road passenger-transport services"},
|
||||
"CPV-77310000": {"scheme": "CPV", "id": "77310000", "description": "Planting and maintenance services of green areas"},
|
||||
"CPV-77314000": {"scheme": "CPV", "id": "77314000", "description": "Grounds maintenance services"}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user