Naming, layout and comments: descriptive full-word names, one item per line in wrapped calls and literals, and doc comments on exported names

Short and single-letter names in application code become full words (tests
keep theirs). Wrapped calls, signatures and composite literals put one item
per line, and one-line function bodies are expanded. Exported names and
groups of constants get doc comments, and comments refer to the renamed
identifiers.
This commit is contained in:
bwilliams committed 2026-10-15 16:12:56 +01:00
1 parent 043e307663
commit 465358609a
78 files changed
+2374 -1402

No files matched your search

+60 -45
View File
@@ -62,15 +62,15 @@ func LoadCatalog() (*Catalog, error) {
}
// parseCatalog decodes and validates catalog JSON; name labels any error.
func parseCatalog(b []byte, name string) (*Catalog, error) {
var c Catalog
if err := json.Unmarshal(b, &c); err != nil {
func parseCatalog(data []byte, name string) (*Catalog, error) {
var catalog Catalog
if err := json.Unmarshal(data, &catalog); err != nil {
return nil, fmt.Errorf("decoding %s: %w", name, err)
}
if err := c.validate(); err != nil {
if err := catalog.validate(); err != nil {
return nil, fmt.Errorf("invalid %s: %w", name, err)
}
return &c, nil
return &catalog, nil
}
// validate checks that every pool has at least one entry and that every
@@ -83,11 +83,26 @@ func (c *Catalog) validate() error {
name string
n int
}{
{"buyers", len(c.Buyers)},
{"suppliers", len(c.Suppliers)},
{"subjects", len(c.Subjects)},
{"methods", len(c.Methods)},
{"classifications", len(c.Classifications)},
{
name: "buyers",
n: len(c.Buyers),
},
{
name: "suppliers",
n: len(c.Suppliers),
},
{
name: "subjects",
n: len(c.Subjects),
},
{
name: "methods",
n: len(c.Methods),
},
{
name: "classifications",
n: len(c.Classifications),
},
} {
if pool.n == 0 {
errs = append(errs, fmt.Errorf("no %s", pool.name))
@@ -96,25 +111,25 @@ func (c *Catalog) validate() error {
missing := map[string]bool{}
check := func(keys ...string) {
for _, k := range keys {
if _, ok := c.Classifications[k]; k != "" && !ok {
missing[k] = true
for _, key := range keys {
if _, ok := c.Classifications[key]; key != "" && !ok {
missing[key] = true
}
}
}
for _, s := range c.Subjects {
check(s.Classification)
check(s.AdditionalClassifications...)
for _, it := range s.Items {
check(it.AdditionalClassifications...)
for _, subject := range c.Subjects {
check(subject.Classification)
check(subject.AdditionalClassifications...)
for _, item := range subject.Items {
check(item.AdditionalClassifications...)
}
}
if len(missing) > 0 {
keys := make([]string, 0, len(missing))
for k := range missing {
keys = append(keys, k)
for key := range missing {
keys = append(keys, key)
}
sort.Strings(keys)
if len(keys) > 5 {
@@ -131,39 +146,39 @@ func (c *Catalog) validate() error {
// ---------- picking (always returns a copy the caller may modify) ----------
func (c *Catalog) buyer(r *rand.Rand) ocds.Organization {
return clone(c.Buyers[r.Intn(len(c.Buyers))])
func (c *Catalog) buyer(rng *rand.Rand) ocds.Organization {
return clone(c.Buyers[rng.Intn(len(c.Buyers))])
}
func (c *Catalog) supplier(r *rand.Rand) ocds.Organization {
return clone(c.Suppliers[r.Intn(len(c.Suppliers))])
func (c *Catalog) supplier(rng *rand.Rand) ocds.Organization {
return clone(c.Suppliers[rng.Intn(len(c.Suppliers))])
}
func (c *Catalog) method(r *rand.Rand) Method {
return c.Methods[r.Intn(len(c.Methods))]
func (c *Catalog) method(rng *rand.Rand) Method {
return c.Methods[rng.Intn(len(c.Methods))]
}
// subject picks a subject and returns it as an ocds.Tender, with its
// classification keys resolved to full classifications.
func (c *Catalog) subject(r *rand.Rand) ocds.Tender {
s := clone(c.Subjects[r.Intn(len(c.Subjects))])
t := s.Tender
t.Classification = c.resolveOne(s.Classification)
t.AdditionalClassifications = c.resolve(s.AdditionalClassifications)
t.Items = nil
for _, it := range s.Items {
item := it.Item
item.AdditionalClassifications = c.resolve(it.AdditionalClassifications)
t.Items = append(t.Items, item)
func (c *Catalog) subject(rng *rand.Rand) ocds.Tender {
subject := clone(c.Subjects[rng.Intn(len(c.Subjects))])
tender := subject.Tender
tender.Classification = c.resolveOne(subject.Classification)
tender.AdditionalClassifications = c.resolve(subject.AdditionalClassifications)
tender.Items = nil
for _, subjectItem := range subject.Items {
item := subjectItem.Item
item.AdditionalClassifications = c.resolve(subjectItem.AdditionalClassifications)
tender.Items = append(tender.Items, item)
}
return t
return tender
}
// resolveOne returns a copy of the classification with key, or nil if there
// is none.
func (c *Catalog) resolveOne(key string) *ocds.Classification {
if cl, ok := c.Classifications[key]; ok {
return &cl
if classification, ok := c.Classifications[key]; ok {
return &classification
}
return nil
}
@@ -172,20 +187,20 @@ func (c *Catalog) resolveOne(key string) *ocds.Classification {
// that have none.
func (c *Catalog) resolve(keys []string) []ocds.Classification {
var out []ocds.Classification
for _, k := range keys {
if cl, ok := c.Classifications[k]; ok {
out = append(out, cl)
for _, key := range keys {
if classification, ok := c.Classifications[key]; ok {
out = append(out, classification)
}
}
return out
}
// clone deep-copies a value through JSON, which is all the generated types need.
func clone[T any](v T) T {
func clone[T any](value T) T {
var out T
b, err := json.Marshal(v)
data, err := json.Marshal(value)
if err == nil {
json.Unmarshal(b, &out) //nolint:errcheck,gosec // b was just marshalled from a T
json.Unmarshal(data, &out) //nolint:errcheck,gosec // b was just marshalled from a T
}
return out
}
+14 -4
View File
@@ -33,8 +33,13 @@ func TestEmbeddedCatalogIsValid(t *testing.T) {
}
if len(cat.Buyers) == 0 || len(cat.Suppliers) == 0 || len(cat.Subjects) == 0 ||
len(cat.Methods) == 0 {
t.Fatalf("embedded catalog has an empty pool: %d buyers, %d suppliers, %d subjects, %d methods",
len(cat.Buyers), len(cat.Suppliers), len(cat.Subjects), len(cat.Methods))
t.Fatalf(
"embedded catalog has an empty pool: %d buyers, %d suppliers, %d subjects, %d methods",
len(cat.Buyers),
len(cat.Suppliers),
len(cat.Subjects),
len(cat.Methods),
)
}
}
@@ -42,8 +47,13 @@ func TestParseCatalog(t *testing.T) {
cat := loadTestCatalog(t)
if len(cat.Buyers) != 2 || len(cat.Suppliers) != 1 || len(cat.Subjects) != 2 ||
len(cat.Methods) != 2 {
t.Fatalf("got %d buyers, %d suppliers, %d subjects, %d methods",
len(cat.Buyers), len(cat.Suppliers), len(cat.Subjects), len(cat.Methods))
t.Fatalf(
"got %d buyers, %d suppliers, %d subjects, %d methods",
len(cat.Buyers),
len(cat.Suppliers),
len(cat.Subjects),
len(cat.Methods),
)
}
if d := cat.Buyers[0].Details; d == nil || len(d.Classifications) != 1 ||
d.Classifications[0].Scheme != "UK_CA_TYPE" {
+1 -2
View File
@@ -6,8 +6,7 @@
package mock
// Files kept in the mock-data directory.
//
// ReleasesFile is created and appended to by the mock.
const (
// ReleasesFile is created and appended to by the mock.
ReleasesFile = "releases.jsonl"
)
+82 -77
View File
@@ -23,12 +23,12 @@ type Generator struct {
opts options
}
// NewGenerator returns a generator drawing from cat and storing releases in
// NewGenerator returns a generator drawing from catalog and storing releases in
// store. The same seed and store contents give the same releases. It logs to
// slog.Default() unless given WithLogger.
func NewGenerator(cat *Catalog, store *Store, seed int64, opts ...Option) *Generator {
func NewGenerator(catalog *Catalog, store *Store, seed int64, opts ...Option) *Generator {
return &Generator{
cat: cat,
cat: catalog,
store: store,
rnd: rand.New(rand.NewSource(seed)), //nolint:gosec // seeded for repeatable mock data
opts: applyOptions(opts),
@@ -56,7 +56,7 @@ func (g *Generator) newProcedure(at time.Time) ocds.Release {
buyer := g.cat.buyer(g.rnd)
ensurePartyID(&buyer)
buyer.Roles = []string{"buyer"}
m := g.cat.method(g.rnd)
method := g.cat.method(g.rnd)
tag, status := "tender", "active"
if g.rnd.Float64() < 0.2 {
@@ -64,38 +64,38 @@ func (g *Generator) newProcedure(at time.Time) ocds.Release {
}
ocid := g.newOCID()
t := g.cat.subject(g.rnd)
t.ID = ocds.ID(ocid)
t.ProcurementMethod = m.Method
t.ProcurementMethodDetails = m.Details
tender := g.cat.subject(g.rnd)
tender.ID = ocds.ID(ocid)
tender.ProcurementMethod = method.Method
tender.ProcurementMethodDetails = method.Details
factor := 0.5 + g.rnd.Float64()*1.5
if t.Value == nil {
if tender.Value == nil {
amount := float64((g.rnd.Intn(5000) + 10) * 1000)
t.Value = &ocds.Value{
tender.Value = &ocds.Value{
Amount: &amount,
Currency: "GBP",
}
} else {
scaleValue(t.Value, factor)
scaleValue(tender.Value, factor)
}
for i := range t.Lots {
scaleValue(t.Lots[i].Value, factor)
for i := range tender.Lots {
scaleValue(tender.Lots[i].Value, factor)
}
if len(t.Lots) == 0 {
v := clone(*t.Value)
t.Lots = []ocds.Lot{{
if len(tender.Lots) == 0 {
value := clone(*tender.Value)
tender.Lots = []ocds.Lot{{
ID: "1",
Title: t.Title,
Value: &v,
Title: tender.Title,
Value: &value,
}}
}
setStatus(&t, status)
setStatus(&tender, status)
if status == "active" {
t.TenderPeriod = &ocds.Period{
tender.TenderPeriod = &ocds.Period{
EndDate: ptr(at.AddDate(0, 0, 30+g.rnd.Intn(30))),
}
}
@@ -111,7 +111,7 @@ func (g *Generator) newProcedure(at time.Time) ocds.Release {
ID: ocds.ID(buyer.ID),
Name: buyer.Name,
},
Tender: &t,
Tender: &tender,
Language: "en",
}
}
@@ -120,13 +120,13 @@ func (g *Generator) newProcedure(at time.Time) ocds.Release {
// tender, and a tender either has its deadline extended (tenderUpdate) or is
// awarded to a supplier from the catalog.
func (g *Generator) advance(prev ocds.Release, at time.Time) ocds.Release {
r := clone(prev)
r.ID = g.store.NextNoticeID(at.Year())
r.Date = at
r.Awards = nil
if r.Tender == nil {
r.Tender = &ocds.Tender{
ID: ocds.ID(r.OCID),
release := clone(prev)
release.ID = g.store.NextNoticeID(at.Year())
release.Date = at
release.Awards = nil
if release.Tender == nil {
release.Tender = &ocds.Tender{
ID: ocds.ID(release.OCID),
}
}
@@ -139,60 +139,60 @@ func (g *Generator) advance(prev ocds.Release, at time.Time) ocds.Release {
next = "tenderUpdate"
}
}
r.Tag = []string{next}
release.Tag = []string{next}
switch next {
case "tender":
setStatus(r.Tender, "active")
r.Tender.TenderPeriod = &ocds.Period{
setStatus(release.Tender, "active")
release.Tender.TenderPeriod = &ocds.Period{
EndDate: ptr(at.AddDate(0, 0, 30+g.rnd.Intn(30))),
}
case "tenderUpdate":
// A typical update: the deadline is extended by two weeks.
end := at.AddDate(0, 0, 30)
if tp := r.Tender.TenderPeriod; tp != nil && tp.EndDate != nil {
end = *tp.EndDate
if period := release.Tender.TenderPeriod; period != nil && period.EndDate != nil {
end = *period.EndDate
}
r.Tender.TenderPeriod = &ocds.Period{
release.Tender.TenderPeriod = &ocds.Period{
EndDate: ptr(end.AddDate(0, 0, 14)),
}
case "award":
setStatus(r.Tender, "complete")
sup := g.cat.supplier(g.rnd)
ensurePartyID(&sup)
sup.Roles = []string{"supplier"}
r.Parties = append(r.Parties, sup)
setStatus(release.Tender, "complete")
supplier := g.cat.supplier(g.rnd)
ensurePartyID(&supplier)
supplier.Roles = []string{"supplier"}
release.Parties = append(release.Parties, supplier)
var value *ocds.Value
if r.Tender.Value != nil {
v := clone(*r.Tender.Value)
scaleValue(&v, 0.8+g.rnd.Float64()*0.2)
value = &v
if release.Tender.Value != nil {
scaled := clone(*release.Tender.Value)
scaleValue(&scaled, 0.8+g.rnd.Float64()*0.2)
value = &scaled
}
var lotIDs []string
for _, lot := range r.Tender.Lots {
for _, lot := range release.Tender.Lots {
if lot.ID != "" {
lotIDs = append(lotIDs, lot.ID)
}
}
r.Awards = []ocds.Award{{
release.Awards = []ocds.Award{{
ID: "1",
Status: "active",
Date: ptr(at),
Value: value,
Suppliers: []ocds.OrganizationReference{
{
ID: ocds.ID(sup.ID),
Name: sup.Name,
ID: ocds.ID(supplier.ID),
Name: supplier.Name,
},
},
RelatedLots: lotIDs,
}}
}
return r
return release
}
// newOCID returns a random ocid not already in the store.
@@ -224,25 +224,25 @@ func (g *Generator) CatchUp(
start = last
}
n := 0
for t := start.Add(g.gap(min, max)); !t.After(now); t = t.Add(g.gap(min, max)) {
if err := g.store.AppendNoSync(g.Make(t)); err != nil {
return n, err
count := 0
for when := start.Add(g.gap(min, max)); !when.After(now); when = when.Add(g.gap(min, max)) {
if err := g.store.AppendNoSync(g.Make(when)); err != nil {
return count, err
}
n++
count++
}
return n, g.store.Sync()
return count, g.store.Sync()
}
// Run publishes a new release after a random gap between min and max until ctx ends.
func (g *Generator) Run(ctx context.Context, min, max time.Duration) {
logger := g.opts.logger("generator")
for {
d := g.gap(min, max)
logger.Info("next release", "in", d.Round(time.Second))
gap := g.gap(min, max)
logger.Info("next release", "in", gap.Round(time.Second))
timer := time.NewTimer(d)
timer := time.NewTimer(gap)
select {
case <-ctx.Done():
timer.Stop()
@@ -250,46 +250,51 @@ func (g *Generator) Run(ctx context.Context, min, max time.Duration) {
case <-timer.C:
}
r := g.Make(time.Now())
if err := g.store.Append(r); err != nil {
release := g.Make(time.Now())
if err := g.store.Append(release); err != nil {
logger.Error("storing release failed", "error", err)
continue
}
logger.Info("published release", "release_id", r.ID, "tag", r.Tag[0], "ocid", r.OCID)
logger.Info(
"published release",
"release_id", release.ID,
"tag", release.Tag[0],
"ocid", release.OCID,
)
}
}
// ---------- helpers ----------
func ptr[T any](v T) *T { return &v }
func ptr[T any](value T) *T { return &value }
func setStatus(t *ocds.Tender, status string) {
t.Status = status
for i := range t.Lots {
t.Lots[i].Status = status
func setStatus(tender *ocds.Tender, status string) {
tender.Status = status
for i := range tender.Lots {
tender.Lots[i].Status = status
}
}
// scaleValue multiplies v's amounts by f, rounded to the nearest 100.
func scaleValue(v *ocds.Value, f float64) {
if v == nil {
// scaleValue multiplies value's amounts by factor, rounded to the nearest 100.
func scaleValue(value *ocds.Value, factor float64) {
if value == nil {
return
}
for _, a := range []**float64{&v.Amount, &v.AmountGross} {
if *a != nil {
*a = ptr(math.Round(**a*f/100) * 100)
for _, amount := range []**float64{&value.Amount, &value.AmountGross} {
if *amount != nil {
*amount = ptr(math.Round(**amount*factor/100) * 100)
}
}
}
// ensurePartyID gives p an id derived from its name if it has none, so the
// ensurePartyID gives party an id derived from its name if it has none, so the
// same organisation always gets the same id.
func ensurePartyID(p *ocds.Organization) {
if p.ID != "" {
func ensurePartyID(party *ocds.Organization) {
if party.ID != "" {
return
}
h := fnv.New32a()
h.Write([]byte(p.Name)) //nolint:gosec // never fails
p.ID = fmt.Sprintf("GB-FTS-%d", h.Sum32()%1000000)
hasher := fnv.New32a()
hasher.Write([]byte(party.Name)) //nolint:gosec // never fails
party.ID = fmt.Sprintf("GB-FTS-%d", hasher.Sum32()%1000000)
}
+50 -14
View File
@@ -17,12 +17,16 @@ func quietLog(t *testing.T) {
t.Helper()
w := log.Writer()
log.SetOutput(io.Discard)
t.Cleanup(func() { log.SetOutput(w) })
t.Cleanup(func() {
log.SetOutput(w)
})
}
func TestGap(t *testing.T) {
//nolint:gosec // G404: seeded math/rand keeps the test deterministic
g := &Generator{rnd: rand.New(rand.NewSource(1))}
g := &Generator{
rnd: rand.New(rand.NewSource(1)),
}
if d := g.gap(time.Second, time.Second); d != time.Second {
t.Errorf("min == max: %s", d)
}
@@ -39,7 +43,10 @@ func TestGap(t *testing.T) {
func TestScaleValue(t *testing.T) {
scaleValue(nil, 2) // must not panic
v := &ocds.Value{Amount: ptr(12345.0), Currency: "GBP"}
v := &ocds.Value{
Amount: ptr(12345.0),
Currency: "GBP",
}
scaleValue(v, 1.5)
if *v.Amount != 18500 { // 18517.5 rounded to the nearest 100
t.Errorf("amount = %v, want 18500", *v.Amount)
@@ -48,7 +55,10 @@ func TestScaleValue(t *testing.T) {
t.Error("nil gross amount was set")
}
v = &ocds.Value{Amount: ptr(1000.0), AmountGross: ptr(1200.0)}
v = &ocds.Value{
Amount: ptr(1000.0),
AmountGross: ptr(1200.0),
}
scaleValue(v, 0.5)
if *v.Amount != 500 || *v.AmountGross != 600 {
t.Errorf("amounts = %v / %v, want 500 / 600", *v.Amount, *v.AmountGross)
@@ -56,15 +66,24 @@ func TestScaleValue(t *testing.T) {
}
func TestEnsurePartyID(t *testing.T) {
p := ocds.Organization{ID: "GB-EXISTING", Name: "Council"}
p := ocds.Organization{
ID: "GB-EXISTING",
Name: "Council",
}
ensurePartyID(&p)
if p.ID != "GB-EXISTING" {
t.Errorf("existing id replaced: %s", p.ID)
}
a := ocds.Organization{Name: "Somewhere Council"}
b := ocds.Organization{Name: "Somewhere Council"}
c := ocds.Organization{Name: "Elsewhere Council"}
a := ocds.Organization{
Name: "Somewhere Council",
}
b := ocds.Organization{
Name: "Somewhere Council",
}
c := ocds.Organization{
Name: "Elsewhere Council",
}
ensurePartyID(&a)
ensurePartyID(&b)
ensurePartyID(&c)
@@ -90,7 +109,13 @@ func TestAdvance(t *testing.T) {
at := time.Date(2026, 10, 8, 12, 0, 0, 0, time.UTC)
// A planning notice without a tender becomes an active tender.
r := g.advance(ocds.Release{OCID: "ocds-x", Tag: []string{"planning"}}, at)
r := g.advance(
ocds.Release{
OCID: "ocds-x",
Tag: []string{"planning"},
},
at,
)
if r.Tag[0] != "tender" || r.Tender == nil || r.Tender.Status != "active" ||
r.Tender.TenderPeriod == nil || r.Tender.TenderPeriod.EndDate == nil {
t.Fatalf("planning -> %v, tender %+v", r.Tag, r.Tender)
@@ -105,10 +130,15 @@ func TestAdvance(t *testing.T) {
OCID: "ocds-y",
Tag: []string{"tender"},
Tender: &ocds.Tender{
ID: "ocds-y",
Value: &ocds.Value{Amount: ptr(10000.0), Currency: "GBP"},
Lots: []ocds.Lot{{ID: "1"}, {ID: ""}},
TenderPeriod: &ocds.Period{EndDate: &end},
ID: "ocds-y",
Value: &ocds.Value{
Amount: ptr(10000.0),
Currency: "GBP",
},
Lots: []ocds.Lot{{ID: "1"}, {ID: ""}},
TenderPeriod: &ocds.Period{
EndDate: &end,
},
},
}
seen := map[string]bool{}
@@ -147,7 +177,13 @@ func TestAdvance(t *testing.T) {
// An update without a deadline gets 30 + 14 days; a tender with no value
// is awarded with no value.
prev = ocds.Release{OCID: "ocds-z", Tag: []string{"tender"}, Tender: &ocds.Tender{ID: "ocds-z"}}
prev = ocds.Release{
OCID: "ocds-z",
Tag: []string{"tender"},
Tender: &ocds.Tender{
ID: "ocds-z",
},
}
for i := 0; i < 50; i++ {
r := g.advance(prev, at)
switch r.Tag[0] {
+80 -61
View File
@@ -14,11 +14,17 @@ import (
"git.avsdev.uk/Go/job-search/internal/mock/findatender/ocds"
)
// Paging and request parameters of the release package endpoint.
const (
// defaultLimit is the page size when the request sets no limit.
defaultLimit = 100
maxLimit = 100
timeLayout = "2006-01-02T15:04:05"
BasePath = "/api/1.0/ocdsReleasePackages"
// maxLimit is the largest page size a request may ask for.
maxLimit = 100
// timeLayout is the documented layout of updatedFrom and updatedTo; see
// parseTime for the others accepted.
timeLayout = "2006-01-02T15:04:05"
// BasePath is the path the endpoint is served at.
BasePath = "/api/1.0/ocdsReleasePackages"
)
// server serves release packages from a Store; see NewHandler.
@@ -36,10 +42,18 @@ type server struct {
// Stored release dates are UTC; comparisons are on absolute instants. It
// logs to slog.Default() unless given WithLogger.
func NewHandler(
store *Store, baseURL string, loc *time.Location, opts ...Option,
store *Store,
baseURL string,
location *time.Location,
opts ...Option,
) http.HandlerFunc {
s := &server{store: store, baseURL: baseURL, loc: loc, opts: applyOptions(opts)}
return s.handle
handler := &server{
store: store,
baseURL: baseURL,
loc: location,
opts: applyOptions(opts),
}
return handler.handle
}
// encodeCursor makes the opaque cursor for a next link that resumes at store
@@ -50,18 +64,18 @@ func encodeCursor(i int) string {
// decodeCursor returns the release index in a cursor made by encodeCursor.
// Anything else, including a negative index, is an error.
func decodeCursor(c string) (int, error) {
b, err := base64.RawURLEncoding.DecodeString(c)
if err != nil || !strings.HasPrefix(string(b), "idx:") {
func decodeCursor(cursor string) (int, error) {
decoded, err := base64.RawURLEncoding.DecodeString(cursor)
if err != nil || !strings.HasPrefix(string(decoded), "idx:") {
return 0, fmt.Errorf("invalid cursor")
}
n, err := strconv.Atoi(strings.TrimPrefix(string(b), "idx:"))
if err != nil || n < 0 {
index, err := strconv.Atoi(strings.TrimPrefix(string(decoded), "idx:"))
if err != nil || index < 0 {
return 0, fmt.Errorf("invalid cursor")
}
return n, nil
return index, nil
}
func badRequest(w http.ResponseWriter, msg string) {
@@ -72,20 +86,20 @@ func badRequest(w http.ResponseWriter, msg string) {
}
// parseTime reads a query timestamp. Values without an offset (the documented
// yyyy-MM-ddTHH:mm:ss, or a plain date) are local time in loc; values with an
// yyyy-MM-ddTHH:mm:ss, or a plain date) are local time in location; values with an
// offset or Z (RFC 3339) mean exactly what they say.
func parseTime(v string, loc *time.Location) (time.Time, error) {
if loc == nil {
loc = time.UTC
func parseTime(value string, location *time.Location) (time.Time, error) {
if location == nil {
location = time.UTC
}
if t, err := time.Parse(time.RFC3339, v); err == nil {
return t, nil
if parsed, err := time.Parse(time.RFC3339, value); err == nil {
return parsed, nil
}
for _, layout := range []string{timeLayout, "2006-01-02"} {
if t, err := time.ParseInLocation(layout, v, loc); err == nil {
return t, nil
if parsed, err := time.ParseInLocation(layout, value, location); err == nil {
return parsed, nil
}
}
@@ -107,15 +121,15 @@ func (s *server) handle(w http.ResponseWriter, req *http.Request) {
return
}
q := req.URL.Query()
pq, err := s.parseQuery(q)
query := req.URL.Query()
page, err := s.parseQuery(query)
if err != nil {
badRequest(w, err.Error())
return
}
releases, next, more := s.store.Page(pq.from, pq.upper, pq.start, pq.limit)
pkg := ocds.ReleasePackage{
releases, next, more := s.store.Page(page.from, page.upper, page.start, page.limit)
releasePackage := ocds.ReleasePackage{
URI: s.baseURL + req.URL.RequestURI(),
Version: "1.1",
Extensions: ocds.DeclaredExtensions,
@@ -127,73 +141,78 @@ func (s *server) handle(w http.ResponseWriter, req *http.Request) {
Releases: releases,
}
if more {
pkg.Links = &ocds.ReleasePackageLinks{Next: s.nextLink(q, next)}
releasePackage.Links = &ocds.ReleasePackageLinks{
Next: s.nextLink(query, next),
}
}
w.Header().Set("Content-Type", "application/json; charset=utf-8")
w.Header().Set("Access-Control-Allow-Origin", "*")
enc := json.NewEncoder(w)
enc.SetIndent("", " ")
enc.SetEscapeHTML(false)
if err := enc.Encode(pkg); err != nil {
encoder := json.NewEncoder(w)
encoder.SetIndent("", " ")
encoder.SetEscapeHTML(false)
if err := encoder.Encode(releasePackage); err != nil {
s.opts.logger("handler").Warn("writing response failed", "error", err)
}
}
// parseQuery reads and checks the query parameters. Its errors are meant for
// the client.
func (s *server) parseQuery(q url.Values) (pageQuery, error) {
pq := pageQuery{limit: defaultLimit, upper: time.Now().UTC()}
if v := q.Get("limit"); v != "" {
n, err := strconv.Atoi(v)
if err != nil || n < 1 || n > maxLimit {
return pq, fmt.Errorf("limit must be an integer between 1 and %d", maxLimit)
}
pq.limit = n
func (s *server) parseQuery(query url.Values) (pageQuery, error) {
page := pageQuery{
limit: defaultLimit,
upper: time.Now().UTC(),
}
if v := q.Get("updatedFrom"); v != "" {
t, err := parseTime(v, s.loc)
if value := query.Get("limit"); value != "" {
number, err := strconv.Atoi(value)
if err != nil || number < 1 || number > maxLimit {
return page, fmt.Errorf("limit must be an integer between 1 and %d", maxLimit)
}
page.limit = number
}
if value := query.Get("updatedFrom"); value != "" {
from, err := parseTime(value, s.loc)
if err != nil {
return pq, fmt.Errorf("updatedFrom %w", err)
return page, fmt.Errorf("updatedFrom %w", err)
}
pq.from = t
page.from = from
}
if v := q.Get("updatedTo"); v != "" {
t, err := parseTime(v, s.loc)
if value := query.Get("updatedTo"); value != "" {
to, err := parseTime(value, s.loc)
if err != nil {
return pq, fmt.Errorf("updatedTo %w", err)
return page, fmt.Errorf("updatedTo %w", err)
}
if q.Get("updatedFrom") != "" && t.Before(pq.from) {
return pq, errors.New("updatedTo must not be before updatedFrom")
if query.Get("updatedFrom") != "" && to.Before(page.from) {
return page, errors.New("updatedTo must not be before updatedFrom")
}
if t.Before(pq.upper) {
pq.upper = t
if to.Before(page.upper) {
page.upper = to
}
}
if c := q.Get("cursor"); c != "" {
n, err := decodeCursor(c)
if cursor := query.Get("cursor"); cursor != "" {
index, err := decodeCursor(cursor)
if err != nil {
return pq, err
return page, err
}
pq.start = n
page.start = index
}
return pq, nil
return page, nil
}
// nextLink is the URL of the page starting at release index next: the same
// query with a new cursor.
func (s *server) nextLink(q url.Values, next int) string {
nq := url.Values{}
for k, vs := range q {
if k != "cursor" {
nq[k] = vs
func (s *server) nextLink(query url.Values, next int) string {
nextQuery := url.Values{}
for key, values := range query {
if key != "cursor" {
nextQuery[key] = values
}
}
nq.Set("cursor", encodeCursor(next))
return s.baseURL + BasePath + "?" + nq.Encode()
nextQuery.Set("cursor", encodeCursor(next))
return s.baseURL + BasePath + "?" + nextQuery.Encode()
}
+6 -2
View File
@@ -79,7 +79,9 @@ func openTestStore(t *testing.T) *Store {
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { store.Close() })
t.Cleanup(func() {
store.Close()
})
return store
}
@@ -92,7 +94,9 @@ func TestHandlerRejectsBadRequests(t *testing.T) {
t.Errorf("POST: status %d, want 405", rec.Code)
}
cursor := func(s string) string { return base64.RawURLEncoding.EncodeToString([]byte(s)) }
cursor := func(s string) string {
return base64.RawURLEncoding.EncodeToString([]byte(s))
}
cases := map[string]string{
"limit=0": "limit must be an integer between 1 and 100",
"limit=101": "limit must be an integer between 1 and 100",
+24 -20
View File
@@ -50,46 +50,50 @@ func (rl *RateLimiter) allow(key string) (bool, int) {
now := rl.now().UTC()
hour := now.Truncate(time.Hour)
c, ok := rl.clients[key]
if !ok || !c.hour.Equal(hour) {
client, ok := rl.clients[key]
if !ok || !client.hour.Equal(hour) {
// First request of this clock hour: it anchors the windows.
if !ok && len(rl.clients) > 10000 {
rl.prune(hour)
}
c = &clientWindow{hour: hour, anchor: now, windowStart: now}
rl.clients[key] = c
client = &clientWindow{
hour: hour,
anchor: now,
windowStart: now,
}
rl.clients[key] = client
}
// Move to whichever fixed window, counted from the anchor, now falls in.
idx := now.Sub(c.anchor) / rl.window
if ws := c.anchor.Add(idx * rl.window); !ws.Equal(c.windowStart) {
c.windowStart = ws
c.count = 0
index := now.Sub(client.anchor) / rl.window
if windowStart := client.anchor.Add(index * rl.window); !windowStart.Equal(client.windowStart) {
client.windowStart = windowStart
client.count = 0
}
if c.count < rl.limit {
c.count++
if client.count < rl.limit {
client.count++
return true, 0
}
// The next window opens at the end of this one, or at the top of the
// next hour if that comes first (because a new hour re-anchors).
next := c.windowStart.Add(rl.window)
if nh := hour.Add(time.Hour); nh.Before(next) {
next = nh
next := client.windowStart.Add(rl.window)
if nextHour := hour.Add(time.Hour); nextHour.Before(next) {
next = nextHour
}
secs := int((next.Sub(now) + time.Second - 1) / time.Second) // round up
if secs < 1 {
secs = 1
seconds := int((next.Sub(now) + time.Second - 1) / time.Second) // round up
if seconds < 1 {
seconds = 1
}
return false, secs
return false, seconds
}
// prune drops clients whose state belongs to an earlier hour.
func (rl *RateLimiter) prune(hour time.Time) {
for k, c := range rl.clients {
if c.hour.Before(hour) {
delete(rl.clients, k)
for key, client := range rl.clients {
if client.hour.Before(hour) {
delete(rl.clients, key)
}
}
}
+9 -3
View File
@@ -12,7 +12,9 @@ import (
func TestRateLimitWindows(t *testing.T) {
now := time.Date(2026, 10, 6, 10, 3, 20, 0, time.UTC) // first request of the hour
rl := NewRateLimiter(12, 5*time.Minute)
rl.now = func() time.Time { return now }
rl.now = func() time.Time {
return now
}
for i := 0; i < 12; i++ {
if ok, _ := rl.allow("a"); !ok {
@@ -63,7 +65,9 @@ func TestClientKey(t *testing.T) {
func TestRateLimitMiddleware(t *testing.T) {
now := time.Date(2026, 10, 6, 10, 0, 0, 0, time.UTC)
rl := NewRateLimiter(2, 5*time.Minute)
rl.now = func() time.Time { return now }
rl.now = func() time.Time {
return now
}
calls := 0
h := rl.Middleware(func(w http.ResponseWriter, req *http.Request) {
@@ -108,7 +112,9 @@ func TestRateLimitMiddleware(t *testing.T) {
func TestRateLimitPrunesStaleClients(t *testing.T) {
now := time.Date(2026, 10, 6, 10, 30, 0, 0, time.UTC)
rl := NewRateLimiter(1, time.Minute)
rl.now = func() time.Time { return now }
rl.now = func() time.Time {
return now
}
for i := 0; i <= 10000; i++ {
rl.allow(strconv.Itoa(i))
+61 -59
View File
@@ -43,7 +43,7 @@ type Store struct {
// left incomplete by a crash is ended so new ones start on a fresh line.
// It logs to slog.Default() unless given WithLogger.
func OpenStore(path string, opts ...Option) (*Store, error) {
s := &Store{
store := &Store{
latest: map[string]int{},
lastNum: map[int]int{},
openPos: map[string]int{},
@@ -53,57 +53,57 @@ func OpenStore(path string, opts ...Option) (*Store, error) {
return nil, err
}
releases, err := readReleases(s.opts.logger("store"), path)
releases, err := readReleases(store.opts.logger("store"), path)
if err != nil {
return nil, err
}
// The file is written in order, but sort anyway in case it was hand-edited.
sort.SliceStable(releases, func(i, j int) bool {
return releases[i].Date.Before(releases[j].Date)
sort.SliceStable(releases, func(i, other int) bool {
return releases[i].Date.Before(releases[other].Date)
})
for _, r := range releases {
s.releases = append(s.releases, r)
s.times = append(s.times, r.Date)
for _, release := range releases {
store.releases = append(store.releases, release)
store.times = append(store.times, release.Date)
}
for i, r := range s.releases {
s.index(i, r)
for i, release := range store.releases {
store.index(i, release)
}
if s.f, err = openForAppend(path); err != nil {
if store.f, err = openForAppend(path); err != nil {
return nil, err
}
return s, nil
return store, nil
}
// readReleases reads the releases stored at path, skipping blank lines and
// lines that can't be decoded (which are logged to logger). A missing file
// holds no releases.
func readReleases(logger *slog.Logger, path string) ([]ocds.Release, error) {
in, err := os.Open(filepath.Clean(path))
input, err := os.Open(filepath.Clean(path))
if os.IsNotExist(err) {
return nil, nil
}
if err != nil {
return nil, err
}
defer in.Close() //nolint:errcheck // read only
defer input.Close() //nolint:errcheck // read only
var releases []ocds.Release
sc := bufio.NewScanner(in)
sc.Buffer(make([]byte, 1<<20), 64<<20)
for line := 1; sc.Scan(); line++ {
b := bytes.TrimSpace(sc.Bytes())
if len(b) == 0 {
scanner := bufio.NewScanner(input)
scanner.Buffer(make([]byte, 1<<20), 64<<20)
for line := 1; scanner.Scan(); line++ {
data := bytes.TrimSpace(scanner.Bytes())
if len(data) == 0 {
continue
}
var r ocds.Release
if err := json.Unmarshal(b, &r); err != nil {
var release ocds.Release
if err := json.Unmarshal(data, &release); err != nil {
logger.Warn("skipping unreadable line", "line", line, "error", err)
continue
}
releases = append(releases, r)
releases = append(releases, release)
}
if err := sc.Err(); err != nil {
if err := scanner.Err(); err != nil {
return nil, fmt.Errorf("reading %s: %w", path, err)
}
return releases, nil
@@ -113,60 +113,60 @@ func readReleases(logger *slog.Logger, path string) ([]ocds.Release, error) {
// mid-write, it ends the partial record so the next one starts on a fresh
// line.
func openForAppend(path string) (*os.File, error) {
f, err := os.OpenFile(filepath.Clean(path), os.O_CREATE|os.O_RDWR|os.O_APPEND, 0o600)
file, err := os.OpenFile(filepath.Clean(path), os.O_CREATE|os.O_RDWR|os.O_APPEND, 0o600)
if err != nil {
return nil, err
}
if st, err := f.Stat(); err == nil && st.Size() > 0 {
if info, err := file.Stat(); err == nil && info.Size() > 0 {
last := make([]byte, 1)
if _, err := f.ReadAt(last, st.Size()-1); err == nil && last[0] != '\n' {
if _, err := f.Write([]byte("\n")); err != nil {
return nil, errors.Join(fmt.Errorf("repairing %s: %w", path, err), f.Close())
if _, err := file.ReadAt(last, info.Size()-1); err == nil && last[0] != '\n' {
if _, err := file.Write([]byte("\n")); err != nil {
return nil, errors.Join(fmt.Errorf("repairing %s: %w", path, err), file.Close())
}
}
}
return f, nil
return file, nil
}
// index records release r, at position i, in the lookups: the ocid's latest
// index records release, at position i, in the lookups: the ocid's latest
// release, whether its procedure is still open (an award closes it), and the
// highest notice number used each year.
func (s *Store) index(i int, r ocds.Release) {
s.latest[r.OCID] = i
if len(r.Tag) > 0 && r.Tag[0] == "award" {
s.closeProcedure(r.OCID)
} else if _, ok := s.openPos[r.OCID]; !ok {
s.openPos[r.OCID] = len(s.open)
s.open = append(s.open, r.OCID)
func (s *Store) index(i int, release ocds.Release) {
s.latest[release.OCID] = i
if len(release.Tag) > 0 && release.Tag[0] == "award" {
s.closeProcedure(release.OCID)
} else if _, ok := s.openPos[release.OCID]; !ok {
s.openPos[release.OCID] = len(s.open)
s.open = append(s.open, release.OCID)
}
if num, year, ok := strings.Cut(r.ID, "-"); ok {
n, err1 := strconv.Atoi(num)
y, err2 := strconv.Atoi(year)
if err1 == nil && err2 == nil && n > s.lastNum[y] {
s.lastNum[y] = n
if numberText, year, ok := strings.Cut(release.ID, "-"); ok {
number, numberErr := strconv.Atoi(numberText)
yearNumber, yearErr := strconv.Atoi(year)
if numberErr == nil && yearErr == nil && number > s.lastNum[yearNumber] {
s.lastNum[yearNumber] = number
}
}
}
// closeProcedure removes ocid from the open procedures, if it is there.
func (s *Store) closeProcedure(ocid string) {
pos, ok := s.openPos[ocid]
position, ok := s.openPos[ocid]
if !ok {
return
}
last := len(s.open) - 1
s.open[pos] = s.open[last]
s.openPos[s.open[pos]] = pos
s.open[position] = s.open[last]
s.openPos[s.open[position]] = position
s.open = s.open[:last]
delete(s.openPos, ocid)
}
// Append persists a release and flushes it to disk.
func (s *Store) Append(r ocds.Release) error { return s.append(r, true) }
func (s *Store) Append(release ocds.Release) error { return s.append(release, true) }
// AppendNoSync persists a release without an fsync; call Sync afterwards.
// Used for bulk catch-up, where a sync per record would be very slow.
func (s *Store) AppendNoSync(r ocds.Release) error { return s.append(r, false) }
func (s *Store) AppendNoSync(release ocds.Release) error { return s.append(release, false) }
// Sync flushes appended releases to disk. Call it after AppendNoSync.
func (s *Store) Sync() error {
@@ -175,22 +175,22 @@ func (s *Store) Sync() error {
return s.f.Sync()
}
// append stores r, syncing the file to disk if sync is true. A date earlier
// append stores release, syncing the file to disk if sync is true. A date earlier
// than the newest stored release (for example after a clock change) is moved
// up so the order stays valid.
func (s *Store) append(r ocds.Release, sync bool) error {
func (s *Store) append(release ocds.Release, sync bool) error {
s.mu.Lock()
defer s.mu.Unlock()
r.Date = r.Date.UTC()
if n := len(s.times); n > 0 && r.Date.Before(s.times[n-1]) {
r.Date = s.times[n-1]
release.Date = release.Date.UTC()
if count := len(s.times); count > 0 && release.Date.Before(s.times[count-1]) {
release.Date = s.times[count-1]
}
b, err := json.Marshal(r)
data, err := json.Marshal(release)
if err != nil {
return err
}
if _, err := s.f.Write(append(b, '\n')); err != nil {
if _, err := s.f.Write(append(data, '\n')); err != nil {
return err
}
if sync {
@@ -198,9 +198,9 @@ func (s *Store) append(r ocds.Release, sync bool) error {
return err
}
}
s.releases = append(s.releases, r)
s.times = append(s.times, r.Date)
s.index(len(s.releases)-1, r)
s.releases = append(s.releases, release)
s.times = append(s.times, release.Date)
s.index(len(s.releases)-1, release)
return nil
}
@@ -210,7 +210,9 @@ func (s *Store) Page(from, upper time.Time, start, limit int) ([]ocds.Release, i
s.mu.RLock()
defer s.mu.RUnlock()
i := sort.Search(len(s.times), func(i int) bool { return !s.times[i].Before(from) })
i := sort.Search(len(s.times), func(i int) bool {
return !s.times[i].Before(from)
})
if start > i {
i = start
}
@@ -259,11 +261,11 @@ func (s *Store) NextNoticeID(year int) string {
// RandomOpen picks a procedure that has not been awarded and whose latest
// release is after since. Procedures idle since before then are dropped from
// the open set, as if they had been abandoned.
func (s *Store) RandomOpen(r *rand.Rand, since time.Time) (ocds.Release, bool) {
func (s *Store) RandomOpen(rng *rand.Rand, since time.Time) (ocds.Release, bool) {
s.mu.Lock()
defer s.mu.Unlock()
for len(s.open) > 0 {
ocid := s.open[r.Intn(len(s.open))]
ocid := s.open[rng.Intn(len(s.open))]
i := s.latest[ocid]
if s.times[i].After(since) {
return s.releases[i], true
+43 -8
View File
@@ -65,7 +65,11 @@ func TestStorePersistsAndGeneratorProgresses(t *testing.T) {
}
// The HTTP handler pages through the store without gaps or repeats.
srv := &server{store: store2, baseURL: "", loc: time.UTC}
srv := &server{
store: store2,
baseURL: "",
loc: time.UTC,
}
seen := map[string]bool{}
next := BasePath + "?updatedFrom=2026-10-07T00:00:00&limit=7"
for pages := 0; next != ""; pages++ {
@@ -105,7 +109,12 @@ func writeLines(t *testing.T, path string, lines ...string) {
func releaseLine(t *testing.T, ocid, id, tag string, at time.Time) string {
t.Helper()
b, err := json.Marshal(ocds.Release{OCID: ocid, ID: id, Date: at, Tag: []string{tag}})
b, err := json.Marshal(ocds.Release{
OCID: ocid,
ID: id,
Date: at,
Tag: []string{tag},
})
if err != nil {
t.Fatal(err)
}
@@ -117,7 +126,9 @@ func releaseLine(t *testing.T, ocid, id, tag string, at time.Time) string {
func TestOpenStoreRecoversFromMessyFile(t *testing.T) {
path := filepath.Join(t.TempDir(), "r.jsonl")
t0 := time.Date(2026, 10, 8, 9, 0, 0, 0, time.UTC)
writeLines(t, path,
writeLines(
t,
path,
releaseLine(t, "ocds-b", "000007-2026", "tender", t0.Add(2*time.Hour)),
"{not json",
"",
@@ -225,9 +236,19 @@ func TestAppendKeepsDateOrder(t *testing.T) {
}
defer store.Close()
t0 := time.Date(2026, 10, 8, 9, 0, 0, 0, time.UTC)
store.Append(ocds.Release{OCID: "a", ID: "000001-2026", Date: t0, Tag: []string{"tender"}})
store.Append(ocds.Release{
OCID: "a",
ID: "000001-2026",
Date: t0,
Tag: []string{"tender"},
})
store.Append(
ocds.Release{OCID: "b", ID: "000002-2026", Date: t0.Add(-time.Hour), Tag: []string{"tender"}},
ocds.Release{
OCID: "b",
ID: "000002-2026",
Date: t0.Add(-time.Hour),
Tag: []string{"tender"},
},
)
if last, _ := store.Last(); !last.Equal(t0) {
t.Errorf("Last = %s, want %s", last, t0)
@@ -235,7 +256,11 @@ func TestAppendKeepsDateOrder(t *testing.T) {
// Appending to a closed store fails and leaves the store unchanged.
store.Close()
if err := store.Append(ocds.Release{OCID: "c", Date: t0, Tag: []string{"tender"}}); err == nil {
if err := store.Append(ocds.Release{
OCID: "c",
Date: t0,
Tag: []string{"tender"},
}); err == nil {
t.Error("append after close: expected error")
}
if store.Len() != 2 || store.HasOCID("c") {
@@ -251,9 +276,19 @@ func TestRandomOpenDropsStaleProcedures(t *testing.T) {
}
defer store.Close()
t0 := time.Date(2026, 10, 8, 9, 0, 0, 0, time.UTC)
store.Append(ocds.Release{OCID: "old", ID: "000001-2026", Date: t0, Tag: []string{"tender"}})
store.Append(ocds.Release{
OCID: "old",
ID: "000001-2026",
Date: t0,
Tag: []string{"tender"},
})
store.Append(
ocds.Release{OCID: "new", ID: "000002-2026", Date: t0.Add(time.Hour), Tag: []string{"tender"}},
ocds.Release{
OCID: "new",
ID: "000002-2026",
Date: t0.Add(time.Hour),
Tag: []string{"tender"},
},
)
//nolint:gosec // G404: seeded math/rand keeps the test deterministic