Naming, layout and comments: descriptive full-word names, one item per line in wrapped calls and literals, and doc comments on exported names
Short and single-letter names in application code become full words (tests keep theirs). Wrapped calls, signatures and composite literals put one item per line, and one-line function bodies are expanded. Exported names and groups of constants get doc comments, and comments refer to the renamed identifiers.
This commit is contained in:
1 parent
043e307663
commit
465358609a
78 files changed
+2374
-1402
No files matched your search
@@ -62,15 +62,15 @@ func LoadCatalog() (*Catalog, error) {
|
||||
}
|
||||
|
||||
// parseCatalog decodes and validates catalog JSON; name labels any error.
|
||||
func parseCatalog(b []byte, name string) (*Catalog, error) {
|
||||
var c Catalog
|
||||
if err := json.Unmarshal(b, &c); err != nil {
|
||||
func parseCatalog(data []byte, name string) (*Catalog, error) {
|
||||
var catalog Catalog
|
||||
if err := json.Unmarshal(data, &catalog); err != nil {
|
||||
return nil, fmt.Errorf("decoding %s: %w", name, err)
|
||||
}
|
||||
if err := c.validate(); err != nil {
|
||||
if err := catalog.validate(); err != nil {
|
||||
return nil, fmt.Errorf("invalid %s: %w", name, err)
|
||||
}
|
||||
return &c, nil
|
||||
return &catalog, nil
|
||||
}
|
||||
|
||||
// validate checks that every pool has at least one entry and that every
|
||||
@@ -83,11 +83,26 @@ func (c *Catalog) validate() error {
|
||||
name string
|
||||
n int
|
||||
}{
|
||||
{"buyers", len(c.Buyers)},
|
||||
{"suppliers", len(c.Suppliers)},
|
||||
{"subjects", len(c.Subjects)},
|
||||
{"methods", len(c.Methods)},
|
||||
{"classifications", len(c.Classifications)},
|
||||
{
|
||||
name: "buyers",
|
||||
n: len(c.Buyers),
|
||||
},
|
||||
{
|
||||
name: "suppliers",
|
||||
n: len(c.Suppliers),
|
||||
},
|
||||
{
|
||||
name: "subjects",
|
||||
n: len(c.Subjects),
|
||||
},
|
||||
{
|
||||
name: "methods",
|
||||
n: len(c.Methods),
|
||||
},
|
||||
{
|
||||
name: "classifications",
|
||||
n: len(c.Classifications),
|
||||
},
|
||||
} {
|
||||
if pool.n == 0 {
|
||||
errs = append(errs, fmt.Errorf("no %s", pool.name))
|
||||
@@ -96,25 +111,25 @@ func (c *Catalog) validate() error {
|
||||
|
||||
missing := map[string]bool{}
|
||||
check := func(keys ...string) {
|
||||
for _, k := range keys {
|
||||
if _, ok := c.Classifications[k]; k != "" && !ok {
|
||||
missing[k] = true
|
||||
for _, key := range keys {
|
||||
if _, ok := c.Classifications[key]; key != "" && !ok {
|
||||
missing[key] = true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for _, s := range c.Subjects {
|
||||
check(s.Classification)
|
||||
check(s.AdditionalClassifications...)
|
||||
for _, it := range s.Items {
|
||||
check(it.AdditionalClassifications...)
|
||||
for _, subject := range c.Subjects {
|
||||
check(subject.Classification)
|
||||
check(subject.AdditionalClassifications...)
|
||||
for _, item := range subject.Items {
|
||||
check(item.AdditionalClassifications...)
|
||||
}
|
||||
}
|
||||
|
||||
if len(missing) > 0 {
|
||||
keys := make([]string, 0, len(missing))
|
||||
for k := range missing {
|
||||
keys = append(keys, k)
|
||||
for key := range missing {
|
||||
keys = append(keys, key)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
if len(keys) > 5 {
|
||||
@@ -131,39 +146,39 @@ func (c *Catalog) validate() error {
|
||||
|
||||
// ---------- picking (always returns a copy the caller may modify) ----------
|
||||
|
||||
func (c *Catalog) buyer(r *rand.Rand) ocds.Organization {
|
||||
return clone(c.Buyers[r.Intn(len(c.Buyers))])
|
||||
func (c *Catalog) buyer(rng *rand.Rand) ocds.Organization {
|
||||
return clone(c.Buyers[rng.Intn(len(c.Buyers))])
|
||||
}
|
||||
|
||||
func (c *Catalog) supplier(r *rand.Rand) ocds.Organization {
|
||||
return clone(c.Suppliers[r.Intn(len(c.Suppliers))])
|
||||
func (c *Catalog) supplier(rng *rand.Rand) ocds.Organization {
|
||||
return clone(c.Suppliers[rng.Intn(len(c.Suppliers))])
|
||||
}
|
||||
|
||||
func (c *Catalog) method(r *rand.Rand) Method {
|
||||
return c.Methods[r.Intn(len(c.Methods))]
|
||||
func (c *Catalog) method(rng *rand.Rand) Method {
|
||||
return c.Methods[rng.Intn(len(c.Methods))]
|
||||
}
|
||||
|
||||
// subject picks a subject and returns it as an ocds.Tender, with its
|
||||
// classification keys resolved to full classifications.
|
||||
func (c *Catalog) subject(r *rand.Rand) ocds.Tender {
|
||||
s := clone(c.Subjects[r.Intn(len(c.Subjects))])
|
||||
t := s.Tender
|
||||
t.Classification = c.resolveOne(s.Classification)
|
||||
t.AdditionalClassifications = c.resolve(s.AdditionalClassifications)
|
||||
t.Items = nil
|
||||
for _, it := range s.Items {
|
||||
item := it.Item
|
||||
item.AdditionalClassifications = c.resolve(it.AdditionalClassifications)
|
||||
t.Items = append(t.Items, item)
|
||||
func (c *Catalog) subject(rng *rand.Rand) ocds.Tender {
|
||||
subject := clone(c.Subjects[rng.Intn(len(c.Subjects))])
|
||||
tender := subject.Tender
|
||||
tender.Classification = c.resolveOne(subject.Classification)
|
||||
tender.AdditionalClassifications = c.resolve(subject.AdditionalClassifications)
|
||||
tender.Items = nil
|
||||
for _, subjectItem := range subject.Items {
|
||||
item := subjectItem.Item
|
||||
item.AdditionalClassifications = c.resolve(subjectItem.AdditionalClassifications)
|
||||
tender.Items = append(tender.Items, item)
|
||||
}
|
||||
return t
|
||||
return tender
|
||||
}
|
||||
|
||||
// resolveOne returns a copy of the classification with key, or nil if there
|
||||
// is none.
|
||||
func (c *Catalog) resolveOne(key string) *ocds.Classification {
|
||||
if cl, ok := c.Classifications[key]; ok {
|
||||
return &cl
|
||||
if classification, ok := c.Classifications[key]; ok {
|
||||
return &classification
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -172,20 +187,20 @@ func (c *Catalog) resolveOne(key string) *ocds.Classification {
|
||||
// that have none.
|
||||
func (c *Catalog) resolve(keys []string) []ocds.Classification {
|
||||
var out []ocds.Classification
|
||||
for _, k := range keys {
|
||||
if cl, ok := c.Classifications[k]; ok {
|
||||
out = append(out, cl)
|
||||
for _, key := range keys {
|
||||
if classification, ok := c.Classifications[key]; ok {
|
||||
out = append(out, classification)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// clone deep-copies a value through JSON, which is all the generated types need.
|
||||
func clone[T any](v T) T {
|
||||
func clone[T any](value T) T {
|
||||
var out T
|
||||
b, err := json.Marshal(v)
|
||||
data, err := json.Marshal(value)
|
||||
if err == nil {
|
||||
json.Unmarshal(b, &out) //nolint:errcheck,gosec // b was just marshalled from a T
|
||||
json.Unmarshal(data, &out) //nolint:errcheck,gosec // b was just marshalled from a T
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -33,8 +33,13 @@ func TestEmbeddedCatalogIsValid(t *testing.T) {
|
||||
}
|
||||
if len(cat.Buyers) == 0 || len(cat.Suppliers) == 0 || len(cat.Subjects) == 0 ||
|
||||
len(cat.Methods) == 0 {
|
||||
t.Fatalf("embedded catalog has an empty pool: %d buyers, %d suppliers, %d subjects, %d methods",
|
||||
len(cat.Buyers), len(cat.Suppliers), len(cat.Subjects), len(cat.Methods))
|
||||
t.Fatalf(
|
||||
"embedded catalog has an empty pool: %d buyers, %d suppliers, %d subjects, %d methods",
|
||||
len(cat.Buyers),
|
||||
len(cat.Suppliers),
|
||||
len(cat.Subjects),
|
||||
len(cat.Methods),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -42,8 +47,13 @@ func TestParseCatalog(t *testing.T) {
|
||||
cat := loadTestCatalog(t)
|
||||
if len(cat.Buyers) != 2 || len(cat.Suppliers) != 1 || len(cat.Subjects) != 2 ||
|
||||
len(cat.Methods) != 2 {
|
||||
t.Fatalf("got %d buyers, %d suppliers, %d subjects, %d methods",
|
||||
len(cat.Buyers), len(cat.Suppliers), len(cat.Subjects), len(cat.Methods))
|
||||
t.Fatalf(
|
||||
"got %d buyers, %d suppliers, %d subjects, %d methods",
|
||||
len(cat.Buyers),
|
||||
len(cat.Suppliers),
|
||||
len(cat.Subjects),
|
||||
len(cat.Methods),
|
||||
)
|
||||
}
|
||||
if d := cat.Buyers[0].Details; d == nil || len(d.Classifications) != 1 ||
|
||||
d.Classifications[0].Scheme != "UK_CA_TYPE" {
|
||||
|
||||
@@ -6,8 +6,7 @@
|
||||
package mock
|
||||
|
||||
// Files kept in the mock-data directory.
|
||||
//
|
||||
// ReleasesFile is created and appended to by the mock.
|
||||
const (
|
||||
// ReleasesFile is created and appended to by the mock.
|
||||
ReleasesFile = "releases.jsonl"
|
||||
)
|
||||
@@ -23,12 +23,12 @@ type Generator struct {
|
||||
opts options
|
||||
}
|
||||
|
||||
// NewGenerator returns a generator drawing from cat and storing releases in
|
||||
// NewGenerator returns a generator drawing from catalog and storing releases in
|
||||
// store. The same seed and store contents give the same releases. It logs to
|
||||
// slog.Default() unless given WithLogger.
|
||||
func NewGenerator(cat *Catalog, store *Store, seed int64, opts ...Option) *Generator {
|
||||
func NewGenerator(catalog *Catalog, store *Store, seed int64, opts ...Option) *Generator {
|
||||
return &Generator{
|
||||
cat: cat,
|
||||
cat: catalog,
|
||||
store: store,
|
||||
rnd: rand.New(rand.NewSource(seed)), //nolint:gosec // seeded for repeatable mock data
|
||||
opts: applyOptions(opts),
|
||||
@@ -56,7 +56,7 @@ func (g *Generator) newProcedure(at time.Time) ocds.Release {
|
||||
buyer := g.cat.buyer(g.rnd)
|
||||
ensurePartyID(&buyer)
|
||||
buyer.Roles = []string{"buyer"}
|
||||
m := g.cat.method(g.rnd)
|
||||
method := g.cat.method(g.rnd)
|
||||
|
||||
tag, status := "tender", "active"
|
||||
if g.rnd.Float64() < 0.2 {
|
||||
@@ -64,38 +64,38 @@ func (g *Generator) newProcedure(at time.Time) ocds.Release {
|
||||
}
|
||||
ocid := g.newOCID()
|
||||
|
||||
t := g.cat.subject(g.rnd)
|
||||
t.ID = ocds.ID(ocid)
|
||||
t.ProcurementMethod = m.Method
|
||||
t.ProcurementMethodDetails = m.Details
|
||||
tender := g.cat.subject(g.rnd)
|
||||
tender.ID = ocds.ID(ocid)
|
||||
tender.ProcurementMethod = method.Method
|
||||
tender.ProcurementMethodDetails = method.Details
|
||||
|
||||
factor := 0.5 + g.rnd.Float64()*1.5
|
||||
if t.Value == nil {
|
||||
if tender.Value == nil {
|
||||
amount := float64((g.rnd.Intn(5000) + 10) * 1000)
|
||||
t.Value = &ocds.Value{
|
||||
tender.Value = &ocds.Value{
|
||||
Amount: &amount,
|
||||
Currency: "GBP",
|
||||
}
|
||||
} else {
|
||||
scaleValue(t.Value, factor)
|
||||
scaleValue(tender.Value, factor)
|
||||
}
|
||||
|
||||
for i := range t.Lots {
|
||||
scaleValue(t.Lots[i].Value, factor)
|
||||
for i := range tender.Lots {
|
||||
scaleValue(tender.Lots[i].Value, factor)
|
||||
}
|
||||
|
||||
if len(t.Lots) == 0 {
|
||||
v := clone(*t.Value)
|
||||
t.Lots = []ocds.Lot{{
|
||||
if len(tender.Lots) == 0 {
|
||||
value := clone(*tender.Value)
|
||||
tender.Lots = []ocds.Lot{{
|
||||
ID: "1",
|
||||
Title: t.Title,
|
||||
Value: &v,
|
||||
Title: tender.Title,
|
||||
Value: &value,
|
||||
}}
|
||||
}
|
||||
|
||||
setStatus(&t, status)
|
||||
setStatus(&tender, status)
|
||||
if status == "active" {
|
||||
t.TenderPeriod = &ocds.Period{
|
||||
tender.TenderPeriod = &ocds.Period{
|
||||
EndDate: ptr(at.AddDate(0, 0, 30+g.rnd.Intn(30))),
|
||||
}
|
||||
}
|
||||
@@ -111,7 +111,7 @@ func (g *Generator) newProcedure(at time.Time) ocds.Release {
|
||||
ID: ocds.ID(buyer.ID),
|
||||
Name: buyer.Name,
|
||||
},
|
||||
Tender: &t,
|
||||
Tender: &tender,
|
||||
Language: "en",
|
||||
}
|
||||
}
|
||||
@@ -120,13 +120,13 @@ func (g *Generator) newProcedure(at time.Time) ocds.Release {
|
||||
// tender, and a tender either has its deadline extended (tenderUpdate) or is
|
||||
// awarded to a supplier from the catalog.
|
||||
func (g *Generator) advance(prev ocds.Release, at time.Time) ocds.Release {
|
||||
r := clone(prev)
|
||||
r.ID = g.store.NextNoticeID(at.Year())
|
||||
r.Date = at
|
||||
r.Awards = nil
|
||||
if r.Tender == nil {
|
||||
r.Tender = &ocds.Tender{
|
||||
ID: ocds.ID(r.OCID),
|
||||
release := clone(prev)
|
||||
release.ID = g.store.NextNoticeID(at.Year())
|
||||
release.Date = at
|
||||
release.Awards = nil
|
||||
if release.Tender == nil {
|
||||
release.Tender = &ocds.Tender{
|
||||
ID: ocds.ID(release.OCID),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -139,60 +139,60 @@ func (g *Generator) advance(prev ocds.Release, at time.Time) ocds.Release {
|
||||
next = "tenderUpdate"
|
||||
}
|
||||
}
|
||||
r.Tag = []string{next}
|
||||
release.Tag = []string{next}
|
||||
|
||||
switch next {
|
||||
case "tender":
|
||||
setStatus(r.Tender, "active")
|
||||
r.Tender.TenderPeriod = &ocds.Period{
|
||||
setStatus(release.Tender, "active")
|
||||
release.Tender.TenderPeriod = &ocds.Period{
|
||||
EndDate: ptr(at.AddDate(0, 0, 30+g.rnd.Intn(30))),
|
||||
}
|
||||
case "tenderUpdate":
|
||||
// A typical update: the deadline is extended by two weeks.
|
||||
end := at.AddDate(0, 0, 30)
|
||||
if tp := r.Tender.TenderPeriod; tp != nil && tp.EndDate != nil {
|
||||
end = *tp.EndDate
|
||||
if period := release.Tender.TenderPeriod; period != nil && period.EndDate != nil {
|
||||
end = *period.EndDate
|
||||
}
|
||||
r.Tender.TenderPeriod = &ocds.Period{
|
||||
release.Tender.TenderPeriod = &ocds.Period{
|
||||
EndDate: ptr(end.AddDate(0, 0, 14)),
|
||||
}
|
||||
case "award":
|
||||
setStatus(r.Tender, "complete")
|
||||
sup := g.cat.supplier(g.rnd)
|
||||
ensurePartyID(&sup)
|
||||
sup.Roles = []string{"supplier"}
|
||||
r.Parties = append(r.Parties, sup)
|
||||
setStatus(release.Tender, "complete")
|
||||
supplier := g.cat.supplier(g.rnd)
|
||||
ensurePartyID(&supplier)
|
||||
supplier.Roles = []string{"supplier"}
|
||||
release.Parties = append(release.Parties, supplier)
|
||||
|
||||
var value *ocds.Value
|
||||
if r.Tender.Value != nil {
|
||||
v := clone(*r.Tender.Value)
|
||||
scaleValue(&v, 0.8+g.rnd.Float64()*0.2)
|
||||
value = &v
|
||||
if release.Tender.Value != nil {
|
||||
scaled := clone(*release.Tender.Value)
|
||||
scaleValue(&scaled, 0.8+g.rnd.Float64()*0.2)
|
||||
value = &scaled
|
||||
}
|
||||
|
||||
var lotIDs []string
|
||||
for _, lot := range r.Tender.Lots {
|
||||
for _, lot := range release.Tender.Lots {
|
||||
if lot.ID != "" {
|
||||
lotIDs = append(lotIDs, lot.ID)
|
||||
}
|
||||
}
|
||||
|
||||
r.Awards = []ocds.Award{{
|
||||
release.Awards = []ocds.Award{{
|
||||
ID: "1",
|
||||
Status: "active",
|
||||
Date: ptr(at),
|
||||
Value: value,
|
||||
Suppliers: []ocds.OrganizationReference{
|
||||
{
|
||||
ID: ocds.ID(sup.ID),
|
||||
Name: sup.Name,
|
||||
ID: ocds.ID(supplier.ID),
|
||||
Name: supplier.Name,
|
||||
},
|
||||
},
|
||||
RelatedLots: lotIDs,
|
||||
}}
|
||||
}
|
||||
|
||||
return r
|
||||
return release
|
||||
}
|
||||
|
||||
// newOCID returns a random ocid not already in the store.
|
||||
@@ -224,25 +224,25 @@ func (g *Generator) CatchUp(
|
||||
start = last
|
||||
}
|
||||
|
||||
n := 0
|
||||
for t := start.Add(g.gap(min, max)); !t.After(now); t = t.Add(g.gap(min, max)) {
|
||||
if err := g.store.AppendNoSync(g.Make(t)); err != nil {
|
||||
return n, err
|
||||
count := 0
|
||||
for when := start.Add(g.gap(min, max)); !when.After(now); when = when.Add(g.gap(min, max)) {
|
||||
if err := g.store.AppendNoSync(g.Make(when)); err != nil {
|
||||
return count, err
|
||||
}
|
||||
n++
|
||||
count++
|
||||
}
|
||||
|
||||
return n, g.store.Sync()
|
||||
return count, g.store.Sync()
|
||||
}
|
||||
|
||||
// Run publishes a new release after a random gap between min and max until ctx ends.
|
||||
func (g *Generator) Run(ctx context.Context, min, max time.Duration) {
|
||||
logger := g.opts.logger("generator")
|
||||
for {
|
||||
d := g.gap(min, max)
|
||||
logger.Info("next release", "in", d.Round(time.Second))
|
||||
gap := g.gap(min, max)
|
||||
logger.Info("next release", "in", gap.Round(time.Second))
|
||||
|
||||
timer := time.NewTimer(d)
|
||||
timer := time.NewTimer(gap)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
timer.Stop()
|
||||
@@ -250,46 +250,51 @@ func (g *Generator) Run(ctx context.Context, min, max time.Duration) {
|
||||
case <-timer.C:
|
||||
}
|
||||
|
||||
r := g.Make(time.Now())
|
||||
if err := g.store.Append(r); err != nil {
|
||||
release := g.Make(time.Now())
|
||||
if err := g.store.Append(release); err != nil {
|
||||
logger.Error("storing release failed", "error", err)
|
||||
continue
|
||||
}
|
||||
|
||||
logger.Info("published release", "release_id", r.ID, "tag", r.Tag[0], "ocid", r.OCID)
|
||||
logger.Info(
|
||||
"published release",
|
||||
"release_id", release.ID,
|
||||
"tag", release.Tag[0],
|
||||
"ocid", release.OCID,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
// ---------- helpers ----------
|
||||
|
||||
func ptr[T any](v T) *T { return &v }
|
||||
func ptr[T any](value T) *T { return &value }
|
||||
|
||||
func setStatus(t *ocds.Tender, status string) {
|
||||
t.Status = status
|
||||
for i := range t.Lots {
|
||||
t.Lots[i].Status = status
|
||||
func setStatus(tender *ocds.Tender, status string) {
|
||||
tender.Status = status
|
||||
for i := range tender.Lots {
|
||||
tender.Lots[i].Status = status
|
||||
}
|
||||
}
|
||||
|
||||
// scaleValue multiplies v's amounts by f, rounded to the nearest 100.
|
||||
func scaleValue(v *ocds.Value, f float64) {
|
||||
if v == nil {
|
||||
// scaleValue multiplies value's amounts by factor, rounded to the nearest 100.
|
||||
func scaleValue(value *ocds.Value, factor float64) {
|
||||
if value == nil {
|
||||
return
|
||||
}
|
||||
for _, a := range []**float64{&v.Amount, &v.AmountGross} {
|
||||
if *a != nil {
|
||||
*a = ptr(math.Round(**a*f/100) * 100)
|
||||
for _, amount := range []**float64{&value.Amount, &value.AmountGross} {
|
||||
if *amount != nil {
|
||||
*amount = ptr(math.Round(**amount*factor/100) * 100)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ensurePartyID gives p an id derived from its name if it has none, so the
|
||||
// ensurePartyID gives party an id derived from its name if it has none, so the
|
||||
// same organisation always gets the same id.
|
||||
func ensurePartyID(p *ocds.Organization) {
|
||||
if p.ID != "" {
|
||||
func ensurePartyID(party *ocds.Organization) {
|
||||
if party.ID != "" {
|
||||
return
|
||||
}
|
||||
h := fnv.New32a()
|
||||
h.Write([]byte(p.Name)) //nolint:gosec // never fails
|
||||
p.ID = fmt.Sprintf("GB-FTS-%d", h.Sum32()%1000000)
|
||||
hasher := fnv.New32a()
|
||||
hasher.Write([]byte(party.Name)) //nolint:gosec // never fails
|
||||
party.ID = fmt.Sprintf("GB-FTS-%d", hasher.Sum32()%1000000)
|
||||
}
|
||||
@@ -17,12 +17,16 @@ func quietLog(t *testing.T) {
|
||||
t.Helper()
|
||||
w := log.Writer()
|
||||
log.SetOutput(io.Discard)
|
||||
t.Cleanup(func() { log.SetOutput(w) })
|
||||
t.Cleanup(func() {
|
||||
log.SetOutput(w)
|
||||
})
|
||||
}
|
||||
|
||||
func TestGap(t *testing.T) {
|
||||
//nolint:gosec // G404: seeded math/rand keeps the test deterministic
|
||||
g := &Generator{rnd: rand.New(rand.NewSource(1))}
|
||||
g := &Generator{
|
||||
rnd: rand.New(rand.NewSource(1)),
|
||||
}
|
||||
if d := g.gap(time.Second, time.Second); d != time.Second {
|
||||
t.Errorf("min == max: %s", d)
|
||||
}
|
||||
@@ -39,7 +43,10 @@ func TestGap(t *testing.T) {
|
||||
func TestScaleValue(t *testing.T) {
|
||||
scaleValue(nil, 2) // must not panic
|
||||
|
||||
v := &ocds.Value{Amount: ptr(12345.0), Currency: "GBP"}
|
||||
v := &ocds.Value{
|
||||
Amount: ptr(12345.0),
|
||||
Currency: "GBP",
|
||||
}
|
||||
scaleValue(v, 1.5)
|
||||
if *v.Amount != 18500 { // 18517.5 rounded to the nearest 100
|
||||
t.Errorf("amount = %v, want 18500", *v.Amount)
|
||||
@@ -48,7 +55,10 @@ func TestScaleValue(t *testing.T) {
|
||||
t.Error("nil gross amount was set")
|
||||
}
|
||||
|
||||
v = &ocds.Value{Amount: ptr(1000.0), AmountGross: ptr(1200.0)}
|
||||
v = &ocds.Value{
|
||||
Amount: ptr(1000.0),
|
||||
AmountGross: ptr(1200.0),
|
||||
}
|
||||
scaleValue(v, 0.5)
|
||||
if *v.Amount != 500 || *v.AmountGross != 600 {
|
||||
t.Errorf("amounts = %v / %v, want 500 / 600", *v.Amount, *v.AmountGross)
|
||||
@@ -56,15 +66,24 @@ func TestScaleValue(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestEnsurePartyID(t *testing.T) {
|
||||
p := ocds.Organization{ID: "GB-EXISTING", Name: "Council"}
|
||||
p := ocds.Organization{
|
||||
ID: "GB-EXISTING",
|
||||
Name: "Council",
|
||||
}
|
||||
ensurePartyID(&p)
|
||||
if p.ID != "GB-EXISTING" {
|
||||
t.Errorf("existing id replaced: %s", p.ID)
|
||||
}
|
||||
|
||||
a := ocds.Organization{Name: "Somewhere Council"}
|
||||
b := ocds.Organization{Name: "Somewhere Council"}
|
||||
c := ocds.Organization{Name: "Elsewhere Council"}
|
||||
a := ocds.Organization{
|
||||
Name: "Somewhere Council",
|
||||
}
|
||||
b := ocds.Organization{
|
||||
Name: "Somewhere Council",
|
||||
}
|
||||
c := ocds.Organization{
|
||||
Name: "Elsewhere Council",
|
||||
}
|
||||
ensurePartyID(&a)
|
||||
ensurePartyID(&b)
|
||||
ensurePartyID(&c)
|
||||
@@ -90,7 +109,13 @@ func TestAdvance(t *testing.T) {
|
||||
at := time.Date(2026, 10, 8, 12, 0, 0, 0, time.UTC)
|
||||
|
||||
// A planning notice without a tender becomes an active tender.
|
||||
r := g.advance(ocds.Release{OCID: "ocds-x", Tag: []string{"planning"}}, at)
|
||||
r := g.advance(
|
||||
ocds.Release{
|
||||
OCID: "ocds-x",
|
||||
Tag: []string{"planning"},
|
||||
},
|
||||
at,
|
||||
)
|
||||
if r.Tag[0] != "tender" || r.Tender == nil || r.Tender.Status != "active" ||
|
||||
r.Tender.TenderPeriod == nil || r.Tender.TenderPeriod.EndDate == nil {
|
||||
t.Fatalf("planning -> %v, tender %+v", r.Tag, r.Tender)
|
||||
@@ -105,10 +130,15 @@ func TestAdvance(t *testing.T) {
|
||||
OCID: "ocds-y",
|
||||
Tag: []string{"tender"},
|
||||
Tender: &ocds.Tender{
|
||||
ID: "ocds-y",
|
||||
Value: &ocds.Value{Amount: ptr(10000.0), Currency: "GBP"},
|
||||
Lots: []ocds.Lot{{ID: "1"}, {ID: ""}},
|
||||
TenderPeriod: &ocds.Period{EndDate: &end},
|
||||
ID: "ocds-y",
|
||||
Value: &ocds.Value{
|
||||
Amount: ptr(10000.0),
|
||||
Currency: "GBP",
|
||||
},
|
||||
Lots: []ocds.Lot{{ID: "1"}, {ID: ""}},
|
||||
TenderPeriod: &ocds.Period{
|
||||
EndDate: &end,
|
||||
},
|
||||
},
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
@@ -147,7 +177,13 @@ func TestAdvance(t *testing.T) {
|
||||
|
||||
// An update without a deadline gets 30 + 14 days; a tender with no value
|
||||
// is awarded with no value.
|
||||
prev = ocds.Release{OCID: "ocds-z", Tag: []string{"tender"}, Tender: &ocds.Tender{ID: "ocds-z"}}
|
||||
prev = ocds.Release{
|
||||
OCID: "ocds-z",
|
||||
Tag: []string{"tender"},
|
||||
Tender: &ocds.Tender{
|
||||
ID: "ocds-z",
|
||||
},
|
||||
}
|
||||
for i := 0; i < 50; i++ {
|
||||
r := g.advance(prev, at)
|
||||
switch r.Tag[0] {
|
||||
|
||||
@@ -14,11 +14,17 @@ import (
|
||||
"git.avsdev.uk/Go/job-search/internal/mock/findatender/ocds"
|
||||
)
|
||||
|
||||
// Paging and request parameters of the release package endpoint.
|
||||
const (
|
||||
// defaultLimit is the page size when the request sets no limit.
|
||||
defaultLimit = 100
|
||||
maxLimit = 100
|
||||
timeLayout = "2006-01-02T15:04:05"
|
||||
BasePath = "/api/1.0/ocdsReleasePackages"
|
||||
// maxLimit is the largest page size a request may ask for.
|
||||
maxLimit = 100
|
||||
// timeLayout is the documented layout of updatedFrom and updatedTo; see
|
||||
// parseTime for the others accepted.
|
||||
timeLayout = "2006-01-02T15:04:05"
|
||||
// BasePath is the path the endpoint is served at.
|
||||
BasePath = "/api/1.0/ocdsReleasePackages"
|
||||
)
|
||||
|
||||
// server serves release packages from a Store; see NewHandler.
|
||||
@@ -36,10 +42,18 @@ type server struct {
|
||||
// Stored release dates are UTC; comparisons are on absolute instants. It
|
||||
// logs to slog.Default() unless given WithLogger.
|
||||
func NewHandler(
|
||||
store *Store, baseURL string, loc *time.Location, opts ...Option,
|
||||
store *Store,
|
||||
baseURL string,
|
||||
location *time.Location,
|
||||
opts ...Option,
|
||||
) http.HandlerFunc {
|
||||
s := &server{store: store, baseURL: baseURL, loc: loc, opts: applyOptions(opts)}
|
||||
return s.handle
|
||||
handler := &server{
|
||||
store: store,
|
||||
baseURL: baseURL,
|
||||
loc: location,
|
||||
opts: applyOptions(opts),
|
||||
}
|
||||
return handler.handle
|
||||
}
|
||||
|
||||
// encodeCursor makes the opaque cursor for a next link that resumes at store
|
||||
@@ -50,18 +64,18 @@ func encodeCursor(i int) string {
|
||||
|
||||
// decodeCursor returns the release index in a cursor made by encodeCursor.
|
||||
// Anything else, including a negative index, is an error.
|
||||
func decodeCursor(c string) (int, error) {
|
||||
b, err := base64.RawURLEncoding.DecodeString(c)
|
||||
if err != nil || !strings.HasPrefix(string(b), "idx:") {
|
||||
func decodeCursor(cursor string) (int, error) {
|
||||
decoded, err := base64.RawURLEncoding.DecodeString(cursor)
|
||||
if err != nil || !strings.HasPrefix(string(decoded), "idx:") {
|
||||
return 0, fmt.Errorf("invalid cursor")
|
||||
}
|
||||
|
||||
n, err := strconv.Atoi(strings.TrimPrefix(string(b), "idx:"))
|
||||
if err != nil || n < 0 {
|
||||
index, err := strconv.Atoi(strings.TrimPrefix(string(decoded), "idx:"))
|
||||
if err != nil || index < 0 {
|
||||
return 0, fmt.Errorf("invalid cursor")
|
||||
}
|
||||
|
||||
return n, nil
|
||||
return index, nil
|
||||
}
|
||||
|
||||
func badRequest(w http.ResponseWriter, msg string) {
|
||||
@@ -72,20 +86,20 @@ func badRequest(w http.ResponseWriter, msg string) {
|
||||
}
|
||||
|
||||
// parseTime reads a query timestamp. Values without an offset (the documented
|
||||
// yyyy-MM-ddTHH:mm:ss, or a plain date) are local time in loc; values with an
|
||||
// yyyy-MM-ddTHH:mm:ss, or a plain date) are local time in location; values with an
|
||||
// offset or Z (RFC 3339) mean exactly what they say.
|
||||
func parseTime(v string, loc *time.Location) (time.Time, error) {
|
||||
if loc == nil {
|
||||
loc = time.UTC
|
||||
func parseTime(value string, location *time.Location) (time.Time, error) {
|
||||
if location == nil {
|
||||
location = time.UTC
|
||||
}
|
||||
|
||||
if t, err := time.Parse(time.RFC3339, v); err == nil {
|
||||
return t, nil
|
||||
if parsed, err := time.Parse(time.RFC3339, value); err == nil {
|
||||
return parsed, nil
|
||||
}
|
||||
|
||||
for _, layout := range []string{timeLayout, "2006-01-02"} {
|
||||
if t, err := time.ParseInLocation(layout, v, loc); err == nil {
|
||||
return t, nil
|
||||
if parsed, err := time.ParseInLocation(layout, value, location); err == nil {
|
||||
return parsed, nil
|
||||
}
|
||||
}
|
||||
|
||||
@@ -107,15 +121,15 @@ func (s *server) handle(w http.ResponseWriter, req *http.Request) {
|
||||
return
|
||||
}
|
||||
|
||||
q := req.URL.Query()
|
||||
pq, err := s.parseQuery(q)
|
||||
query := req.URL.Query()
|
||||
page, err := s.parseQuery(query)
|
||||
if err != nil {
|
||||
badRequest(w, err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
releases, next, more := s.store.Page(pq.from, pq.upper, pq.start, pq.limit)
|
||||
pkg := ocds.ReleasePackage{
|
||||
releases, next, more := s.store.Page(page.from, page.upper, page.start, page.limit)
|
||||
releasePackage := ocds.ReleasePackage{
|
||||
URI: s.baseURL + req.URL.RequestURI(),
|
||||
Version: "1.1",
|
||||
Extensions: ocds.DeclaredExtensions,
|
||||
@@ -127,73 +141,78 @@ func (s *server) handle(w http.ResponseWriter, req *http.Request) {
|
||||
Releases: releases,
|
||||
}
|
||||
if more {
|
||||
pkg.Links = &ocds.ReleasePackageLinks{Next: s.nextLink(q, next)}
|
||||
releasePackage.Links = &ocds.ReleasePackageLinks{
|
||||
Next: s.nextLink(query, next),
|
||||
}
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "application/json; charset=utf-8")
|
||||
w.Header().Set("Access-Control-Allow-Origin", "*")
|
||||
enc := json.NewEncoder(w)
|
||||
enc.SetIndent("", " ")
|
||||
enc.SetEscapeHTML(false)
|
||||
if err := enc.Encode(pkg); err != nil {
|
||||
encoder := json.NewEncoder(w)
|
||||
encoder.SetIndent("", " ")
|
||||
encoder.SetEscapeHTML(false)
|
||||
if err := encoder.Encode(releasePackage); err != nil {
|
||||
s.opts.logger("handler").Warn("writing response failed", "error", err)
|
||||
}
|
||||
}
|
||||
|
||||
// parseQuery reads and checks the query parameters. Its errors are meant for
|
||||
// the client.
|
||||
func (s *server) parseQuery(q url.Values) (pageQuery, error) {
|
||||
pq := pageQuery{limit: defaultLimit, upper: time.Now().UTC()}
|
||||
|
||||
if v := q.Get("limit"); v != "" {
|
||||
n, err := strconv.Atoi(v)
|
||||
if err != nil || n < 1 || n > maxLimit {
|
||||
return pq, fmt.Errorf("limit must be an integer between 1 and %d", maxLimit)
|
||||
}
|
||||
pq.limit = n
|
||||
func (s *server) parseQuery(query url.Values) (pageQuery, error) {
|
||||
page := pageQuery{
|
||||
limit: defaultLimit,
|
||||
upper: time.Now().UTC(),
|
||||
}
|
||||
|
||||
if v := q.Get("updatedFrom"); v != "" {
|
||||
t, err := parseTime(v, s.loc)
|
||||
if value := query.Get("limit"); value != "" {
|
||||
number, err := strconv.Atoi(value)
|
||||
if err != nil || number < 1 || number > maxLimit {
|
||||
return page, fmt.Errorf("limit must be an integer between 1 and %d", maxLimit)
|
||||
}
|
||||
page.limit = number
|
||||
}
|
||||
|
||||
if value := query.Get("updatedFrom"); value != "" {
|
||||
from, err := parseTime(value, s.loc)
|
||||
if err != nil {
|
||||
return pq, fmt.Errorf("updatedFrom %w", err)
|
||||
return page, fmt.Errorf("updatedFrom %w", err)
|
||||
}
|
||||
pq.from = t
|
||||
page.from = from
|
||||
}
|
||||
|
||||
if v := q.Get("updatedTo"); v != "" {
|
||||
t, err := parseTime(v, s.loc)
|
||||
if value := query.Get("updatedTo"); value != "" {
|
||||
to, err := parseTime(value, s.loc)
|
||||
if err != nil {
|
||||
return pq, fmt.Errorf("updatedTo %w", err)
|
||||
return page, fmt.Errorf("updatedTo %w", err)
|
||||
}
|
||||
if q.Get("updatedFrom") != "" && t.Before(pq.from) {
|
||||
return pq, errors.New("updatedTo must not be before updatedFrom")
|
||||
if query.Get("updatedFrom") != "" && to.Before(page.from) {
|
||||
return page, errors.New("updatedTo must not be before updatedFrom")
|
||||
}
|
||||
if t.Before(pq.upper) {
|
||||
pq.upper = t
|
||||
if to.Before(page.upper) {
|
||||
page.upper = to
|
||||
}
|
||||
}
|
||||
|
||||
if c := q.Get("cursor"); c != "" {
|
||||
n, err := decodeCursor(c)
|
||||
if cursor := query.Get("cursor"); cursor != "" {
|
||||
index, err := decodeCursor(cursor)
|
||||
if err != nil {
|
||||
return pq, err
|
||||
return page, err
|
||||
}
|
||||
pq.start = n
|
||||
page.start = index
|
||||
}
|
||||
|
||||
return pq, nil
|
||||
return page, nil
|
||||
}
|
||||
|
||||
// nextLink is the URL of the page starting at release index next: the same
|
||||
// query with a new cursor.
|
||||
func (s *server) nextLink(q url.Values, next int) string {
|
||||
nq := url.Values{}
|
||||
for k, vs := range q {
|
||||
if k != "cursor" {
|
||||
nq[k] = vs
|
||||
func (s *server) nextLink(query url.Values, next int) string {
|
||||
nextQuery := url.Values{}
|
||||
for key, values := range query {
|
||||
if key != "cursor" {
|
||||
nextQuery[key] = values
|
||||
}
|
||||
}
|
||||
nq.Set("cursor", encodeCursor(next))
|
||||
return s.baseURL + BasePath + "?" + nq.Encode()
|
||||
nextQuery.Set("cursor", encodeCursor(next))
|
||||
return s.baseURL + BasePath + "?" + nextQuery.Encode()
|
||||
}
|
||||
@@ -79,7 +79,9 @@ func openTestStore(t *testing.T) *Store {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { store.Close() })
|
||||
t.Cleanup(func() {
|
||||
store.Close()
|
||||
})
|
||||
return store
|
||||
}
|
||||
|
||||
@@ -92,7 +94,9 @@ func TestHandlerRejectsBadRequests(t *testing.T) {
|
||||
t.Errorf("POST: status %d, want 405", rec.Code)
|
||||
}
|
||||
|
||||
cursor := func(s string) string { return base64.RawURLEncoding.EncodeToString([]byte(s)) }
|
||||
cursor := func(s string) string {
|
||||
return base64.RawURLEncoding.EncodeToString([]byte(s))
|
||||
}
|
||||
cases := map[string]string{
|
||||
"limit=0": "limit must be an integer between 1 and 100",
|
||||
"limit=101": "limit must be an integer between 1 and 100",
|
||||
|
||||
@@ -50,46 +50,50 @@ func (rl *RateLimiter) allow(key string) (bool, int) {
|
||||
now := rl.now().UTC()
|
||||
hour := now.Truncate(time.Hour)
|
||||
|
||||
c, ok := rl.clients[key]
|
||||
if !ok || !c.hour.Equal(hour) {
|
||||
client, ok := rl.clients[key]
|
||||
if !ok || !client.hour.Equal(hour) {
|
||||
// First request of this clock hour: it anchors the windows.
|
||||
if !ok && len(rl.clients) > 10000 {
|
||||
rl.prune(hour)
|
||||
}
|
||||
c = &clientWindow{hour: hour, anchor: now, windowStart: now}
|
||||
rl.clients[key] = c
|
||||
client = &clientWindow{
|
||||
hour: hour,
|
||||
anchor: now,
|
||||
windowStart: now,
|
||||
}
|
||||
rl.clients[key] = client
|
||||
}
|
||||
|
||||
// Move to whichever fixed window, counted from the anchor, now falls in.
|
||||
idx := now.Sub(c.anchor) / rl.window
|
||||
if ws := c.anchor.Add(idx * rl.window); !ws.Equal(c.windowStart) {
|
||||
c.windowStart = ws
|
||||
c.count = 0
|
||||
index := now.Sub(client.anchor) / rl.window
|
||||
if windowStart := client.anchor.Add(index * rl.window); !windowStart.Equal(client.windowStart) {
|
||||
client.windowStart = windowStart
|
||||
client.count = 0
|
||||
}
|
||||
|
||||
if c.count < rl.limit {
|
||||
c.count++
|
||||
if client.count < rl.limit {
|
||||
client.count++
|
||||
return true, 0
|
||||
}
|
||||
|
||||
// The next window opens at the end of this one, or at the top of the
|
||||
// next hour if that comes first (because a new hour re-anchors).
|
||||
next := c.windowStart.Add(rl.window)
|
||||
if nh := hour.Add(time.Hour); nh.Before(next) {
|
||||
next = nh
|
||||
next := client.windowStart.Add(rl.window)
|
||||
if nextHour := hour.Add(time.Hour); nextHour.Before(next) {
|
||||
next = nextHour
|
||||
}
|
||||
secs := int((next.Sub(now) + time.Second - 1) / time.Second) // round up
|
||||
if secs < 1 {
|
||||
secs = 1
|
||||
seconds := int((next.Sub(now) + time.Second - 1) / time.Second) // round up
|
||||
if seconds < 1 {
|
||||
seconds = 1
|
||||
}
|
||||
return false, secs
|
||||
return false, seconds
|
||||
}
|
||||
|
||||
// prune drops clients whose state belongs to an earlier hour.
|
||||
func (rl *RateLimiter) prune(hour time.Time) {
|
||||
for k, c := range rl.clients {
|
||||
if c.hour.Before(hour) {
|
||||
delete(rl.clients, k)
|
||||
for key, client := range rl.clients {
|
||||
if client.hour.Before(hour) {
|
||||
delete(rl.clients, key)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -12,7 +12,9 @@ import (
|
||||
func TestRateLimitWindows(t *testing.T) {
|
||||
now := time.Date(2026, 10, 6, 10, 3, 20, 0, time.UTC) // first request of the hour
|
||||
rl := NewRateLimiter(12, 5*time.Minute)
|
||||
rl.now = func() time.Time { return now }
|
||||
rl.now = func() time.Time {
|
||||
return now
|
||||
}
|
||||
|
||||
for i := 0; i < 12; i++ {
|
||||
if ok, _ := rl.allow("a"); !ok {
|
||||
@@ -63,7 +65,9 @@ func TestClientKey(t *testing.T) {
|
||||
func TestRateLimitMiddleware(t *testing.T) {
|
||||
now := time.Date(2026, 10, 6, 10, 0, 0, 0, time.UTC)
|
||||
rl := NewRateLimiter(2, 5*time.Minute)
|
||||
rl.now = func() time.Time { return now }
|
||||
rl.now = func() time.Time {
|
||||
return now
|
||||
}
|
||||
|
||||
calls := 0
|
||||
h := rl.Middleware(func(w http.ResponseWriter, req *http.Request) {
|
||||
@@ -108,7 +112,9 @@ func TestRateLimitMiddleware(t *testing.T) {
|
||||
func TestRateLimitPrunesStaleClients(t *testing.T) {
|
||||
now := time.Date(2026, 10, 6, 10, 30, 0, 0, time.UTC)
|
||||
rl := NewRateLimiter(1, time.Minute)
|
||||
rl.now = func() time.Time { return now }
|
||||
rl.now = func() time.Time {
|
||||
return now
|
||||
}
|
||||
|
||||
for i := 0; i <= 10000; i++ {
|
||||
rl.allow(strconv.Itoa(i))
|
||||
|
||||
@@ -43,7 +43,7 @@ type Store struct {
|
||||
// left incomplete by a crash is ended so new ones start on a fresh line.
|
||||
// It logs to slog.Default() unless given WithLogger.
|
||||
func OpenStore(path string, opts ...Option) (*Store, error) {
|
||||
s := &Store{
|
||||
store := &Store{
|
||||
latest: map[string]int{},
|
||||
lastNum: map[int]int{},
|
||||
openPos: map[string]int{},
|
||||
@@ -53,57 +53,57 @@ func OpenStore(path string, opts ...Option) (*Store, error) {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
releases, err := readReleases(s.opts.logger("store"), path)
|
||||
releases, err := readReleases(store.opts.logger("store"), path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// The file is written in order, but sort anyway in case it was hand-edited.
|
||||
sort.SliceStable(releases, func(i, j int) bool {
|
||||
return releases[i].Date.Before(releases[j].Date)
|
||||
sort.SliceStable(releases, func(i, other int) bool {
|
||||
return releases[i].Date.Before(releases[other].Date)
|
||||
})
|
||||
for _, r := range releases {
|
||||
s.releases = append(s.releases, r)
|
||||
s.times = append(s.times, r.Date)
|
||||
for _, release := range releases {
|
||||
store.releases = append(store.releases, release)
|
||||
store.times = append(store.times, release.Date)
|
||||
}
|
||||
for i, r := range s.releases {
|
||||
s.index(i, r)
|
||||
for i, release := range store.releases {
|
||||
store.index(i, release)
|
||||
}
|
||||
|
||||
if s.f, err = openForAppend(path); err != nil {
|
||||
if store.f, err = openForAppend(path); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return s, nil
|
||||
return store, nil
|
||||
}
|
||||
|
||||
// readReleases reads the releases stored at path, skipping blank lines and
|
||||
// lines that can't be decoded (which are logged to logger). A missing file
|
||||
// holds no releases.
|
||||
func readReleases(logger *slog.Logger, path string) ([]ocds.Release, error) {
|
||||
in, err := os.Open(filepath.Clean(path))
|
||||
input, err := os.Open(filepath.Clean(path))
|
||||
if os.IsNotExist(err) {
|
||||
return nil, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer in.Close() //nolint:errcheck // read only
|
||||
defer input.Close() //nolint:errcheck // read only
|
||||
|
||||
var releases []ocds.Release
|
||||
sc := bufio.NewScanner(in)
|
||||
sc.Buffer(make([]byte, 1<<20), 64<<20)
|
||||
for line := 1; sc.Scan(); line++ {
|
||||
b := bytes.TrimSpace(sc.Bytes())
|
||||
if len(b) == 0 {
|
||||
scanner := bufio.NewScanner(input)
|
||||
scanner.Buffer(make([]byte, 1<<20), 64<<20)
|
||||
for line := 1; scanner.Scan(); line++ {
|
||||
data := bytes.TrimSpace(scanner.Bytes())
|
||||
if len(data) == 0 {
|
||||
continue
|
||||
}
|
||||
var r ocds.Release
|
||||
if err := json.Unmarshal(b, &r); err != nil {
|
||||
var release ocds.Release
|
||||
if err := json.Unmarshal(data, &release); err != nil {
|
||||
logger.Warn("skipping unreadable line", "line", line, "error", err)
|
||||
continue
|
||||
}
|
||||
releases = append(releases, r)
|
||||
releases = append(releases, release)
|
||||
}
|
||||
if err := sc.Err(); err != nil {
|
||||
if err := scanner.Err(); err != nil {
|
||||
return nil, fmt.Errorf("reading %s: %w", path, err)
|
||||
}
|
||||
return releases, nil
|
||||
@@ -113,60 +113,60 @@ func readReleases(logger *slog.Logger, path string) ([]ocds.Release, error) {
|
||||
// mid-write, it ends the partial record so the next one starts on a fresh
|
||||
// line.
|
||||
func openForAppend(path string) (*os.File, error) {
|
||||
f, err := os.OpenFile(filepath.Clean(path), os.O_CREATE|os.O_RDWR|os.O_APPEND, 0o600)
|
||||
file, err := os.OpenFile(filepath.Clean(path), os.O_CREATE|os.O_RDWR|os.O_APPEND, 0o600)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if st, err := f.Stat(); err == nil && st.Size() > 0 {
|
||||
if info, err := file.Stat(); err == nil && info.Size() > 0 {
|
||||
last := make([]byte, 1)
|
||||
if _, err := f.ReadAt(last, st.Size()-1); err == nil && last[0] != '\n' {
|
||||
if _, err := f.Write([]byte("\n")); err != nil {
|
||||
return nil, errors.Join(fmt.Errorf("repairing %s: %w", path, err), f.Close())
|
||||
if _, err := file.ReadAt(last, info.Size()-1); err == nil && last[0] != '\n' {
|
||||
if _, err := file.Write([]byte("\n")); err != nil {
|
||||
return nil, errors.Join(fmt.Errorf("repairing %s: %w", path, err), file.Close())
|
||||
}
|
||||
}
|
||||
}
|
||||
return f, nil
|
||||
return file, nil
|
||||
}
|
||||
|
||||
// index records release r, at position i, in the lookups: the ocid's latest
|
||||
// index records release, at position i, in the lookups: the ocid's latest
|
||||
// release, whether its procedure is still open (an award closes it), and the
|
||||
// highest notice number used each year.
|
||||
func (s *Store) index(i int, r ocds.Release) {
|
||||
s.latest[r.OCID] = i
|
||||
if len(r.Tag) > 0 && r.Tag[0] == "award" {
|
||||
s.closeProcedure(r.OCID)
|
||||
} else if _, ok := s.openPos[r.OCID]; !ok {
|
||||
s.openPos[r.OCID] = len(s.open)
|
||||
s.open = append(s.open, r.OCID)
|
||||
func (s *Store) index(i int, release ocds.Release) {
|
||||
s.latest[release.OCID] = i
|
||||
if len(release.Tag) > 0 && release.Tag[0] == "award" {
|
||||
s.closeProcedure(release.OCID)
|
||||
} else if _, ok := s.openPos[release.OCID]; !ok {
|
||||
s.openPos[release.OCID] = len(s.open)
|
||||
s.open = append(s.open, release.OCID)
|
||||
}
|
||||
if num, year, ok := strings.Cut(r.ID, "-"); ok {
|
||||
n, err1 := strconv.Atoi(num)
|
||||
y, err2 := strconv.Atoi(year)
|
||||
if err1 == nil && err2 == nil && n > s.lastNum[y] {
|
||||
s.lastNum[y] = n
|
||||
if numberText, year, ok := strings.Cut(release.ID, "-"); ok {
|
||||
number, numberErr := strconv.Atoi(numberText)
|
||||
yearNumber, yearErr := strconv.Atoi(year)
|
||||
if numberErr == nil && yearErr == nil && number > s.lastNum[yearNumber] {
|
||||
s.lastNum[yearNumber] = number
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// closeProcedure removes ocid from the open procedures, if it is there.
|
||||
func (s *Store) closeProcedure(ocid string) {
|
||||
pos, ok := s.openPos[ocid]
|
||||
position, ok := s.openPos[ocid]
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
last := len(s.open) - 1
|
||||
s.open[pos] = s.open[last]
|
||||
s.openPos[s.open[pos]] = pos
|
||||
s.open[position] = s.open[last]
|
||||
s.openPos[s.open[position]] = position
|
||||
s.open = s.open[:last]
|
||||
delete(s.openPos, ocid)
|
||||
}
|
||||
|
||||
// Append persists a release and flushes it to disk.
|
||||
func (s *Store) Append(r ocds.Release) error { return s.append(r, true) }
|
||||
func (s *Store) Append(release ocds.Release) error { return s.append(release, true) }
|
||||
|
||||
// AppendNoSync persists a release without an fsync; call Sync afterwards.
|
||||
// Used for bulk catch-up, where a sync per record would be very slow.
|
||||
func (s *Store) AppendNoSync(r ocds.Release) error { return s.append(r, false) }
|
||||
func (s *Store) AppendNoSync(release ocds.Release) error { return s.append(release, false) }
|
||||
|
||||
// Sync flushes appended releases to disk. Call it after AppendNoSync.
|
||||
func (s *Store) Sync() error {
|
||||
@@ -175,22 +175,22 @@ func (s *Store) Sync() error {
|
||||
return s.f.Sync()
|
||||
}
|
||||
|
||||
// append stores r, syncing the file to disk if sync is true. A date earlier
|
||||
// append stores release, syncing the file to disk if sync is true. A date earlier
|
||||
// than the newest stored release (for example after a clock change) is moved
|
||||
// up so the order stays valid.
|
||||
func (s *Store) append(r ocds.Release, sync bool) error {
|
||||
func (s *Store) append(release ocds.Release, sync bool) error {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
|
||||
r.Date = r.Date.UTC()
|
||||
if n := len(s.times); n > 0 && r.Date.Before(s.times[n-1]) {
|
||||
r.Date = s.times[n-1]
|
||||
release.Date = release.Date.UTC()
|
||||
if count := len(s.times); count > 0 && release.Date.Before(s.times[count-1]) {
|
||||
release.Date = s.times[count-1]
|
||||
}
|
||||
b, err := json.Marshal(r)
|
||||
data, err := json.Marshal(release)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := s.f.Write(append(b, '\n')); err != nil {
|
||||
if _, err := s.f.Write(append(data, '\n')); err != nil {
|
||||
return err
|
||||
}
|
||||
if sync {
|
||||
@@ -198,9 +198,9 @@ func (s *Store) append(r ocds.Release, sync bool) error {
|
||||
return err
|
||||
}
|
||||
}
|
||||
s.releases = append(s.releases, r)
|
||||
s.times = append(s.times, r.Date)
|
||||
s.index(len(s.releases)-1, r)
|
||||
s.releases = append(s.releases, release)
|
||||
s.times = append(s.times, release.Date)
|
||||
s.index(len(s.releases)-1, release)
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -210,7 +210,9 @@ func (s *Store) Page(from, upper time.Time, start, limit int) ([]ocds.Release, i
|
||||
s.mu.RLock()
|
||||
defer s.mu.RUnlock()
|
||||
|
||||
i := sort.Search(len(s.times), func(i int) bool { return !s.times[i].Before(from) })
|
||||
i := sort.Search(len(s.times), func(i int) bool {
|
||||
return !s.times[i].Before(from)
|
||||
})
|
||||
if start > i {
|
||||
i = start
|
||||
}
|
||||
@@ -259,11 +261,11 @@ func (s *Store) NextNoticeID(year int) string {
|
||||
// RandomOpen picks a procedure that has not been awarded and whose latest
|
||||
// release is after since. Procedures idle since before then are dropped from
|
||||
// the open set, as if they had been abandoned.
|
||||
func (s *Store) RandomOpen(r *rand.Rand, since time.Time) (ocds.Release, bool) {
|
||||
func (s *Store) RandomOpen(rng *rand.Rand, since time.Time) (ocds.Release, bool) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
for len(s.open) > 0 {
|
||||
ocid := s.open[r.Intn(len(s.open))]
|
||||
ocid := s.open[rng.Intn(len(s.open))]
|
||||
i := s.latest[ocid]
|
||||
if s.times[i].After(since) {
|
||||
return s.releases[i], true
|
||||
|
||||
@@ -65,7 +65,11 @@ func TestStorePersistsAndGeneratorProgresses(t *testing.T) {
|
||||
}
|
||||
|
||||
// The HTTP handler pages through the store without gaps or repeats.
|
||||
srv := &server{store: store2, baseURL: "", loc: time.UTC}
|
||||
srv := &server{
|
||||
store: store2,
|
||||
baseURL: "",
|
||||
loc: time.UTC,
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
next := BasePath + "?updatedFrom=2026-10-07T00:00:00&limit=7"
|
||||
for pages := 0; next != ""; pages++ {
|
||||
@@ -105,7 +109,12 @@ func writeLines(t *testing.T, path string, lines ...string) {
|
||||
|
||||
func releaseLine(t *testing.T, ocid, id, tag string, at time.Time) string {
|
||||
t.Helper()
|
||||
b, err := json.Marshal(ocds.Release{OCID: ocid, ID: id, Date: at, Tag: []string{tag}})
|
||||
b, err := json.Marshal(ocds.Release{
|
||||
OCID: ocid,
|
||||
ID: id,
|
||||
Date: at,
|
||||
Tag: []string{tag},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -117,7 +126,9 @@ func releaseLine(t *testing.T, ocid, id, tag string, at time.Time) string {
|
||||
func TestOpenStoreRecoversFromMessyFile(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "r.jsonl")
|
||||
t0 := time.Date(2026, 10, 8, 9, 0, 0, 0, time.UTC)
|
||||
writeLines(t, path,
|
||||
writeLines(
|
||||
t,
|
||||
path,
|
||||
releaseLine(t, "ocds-b", "000007-2026", "tender", t0.Add(2*time.Hour)),
|
||||
"{not json",
|
||||
"",
|
||||
@@ -225,9 +236,19 @@ func TestAppendKeepsDateOrder(t *testing.T) {
|
||||
}
|
||||
defer store.Close()
|
||||
t0 := time.Date(2026, 10, 8, 9, 0, 0, 0, time.UTC)
|
||||
store.Append(ocds.Release{OCID: "a", ID: "000001-2026", Date: t0, Tag: []string{"tender"}})
|
||||
store.Append(ocds.Release{
|
||||
OCID: "a",
|
||||
ID: "000001-2026",
|
||||
Date: t0,
|
||||
Tag: []string{"tender"},
|
||||
})
|
||||
store.Append(
|
||||
ocds.Release{OCID: "b", ID: "000002-2026", Date: t0.Add(-time.Hour), Tag: []string{"tender"}},
|
||||
ocds.Release{
|
||||
OCID: "b",
|
||||
ID: "000002-2026",
|
||||
Date: t0.Add(-time.Hour),
|
||||
Tag: []string{"tender"},
|
||||
},
|
||||
)
|
||||
if last, _ := store.Last(); !last.Equal(t0) {
|
||||
t.Errorf("Last = %s, want %s", last, t0)
|
||||
@@ -235,7 +256,11 @@ func TestAppendKeepsDateOrder(t *testing.T) {
|
||||
|
||||
// Appending to a closed store fails and leaves the store unchanged.
|
||||
store.Close()
|
||||
if err := store.Append(ocds.Release{OCID: "c", Date: t0, Tag: []string{"tender"}}); err == nil {
|
||||
if err := store.Append(ocds.Release{
|
||||
OCID: "c",
|
||||
Date: t0,
|
||||
Tag: []string{"tender"},
|
||||
}); err == nil {
|
||||
t.Error("append after close: expected error")
|
||||
}
|
||||
if store.Len() != 2 || store.HasOCID("c") {
|
||||
@@ -251,9 +276,19 @@ func TestRandomOpenDropsStaleProcedures(t *testing.T) {
|
||||
}
|
||||
defer store.Close()
|
||||
t0 := time.Date(2026, 10, 8, 9, 0, 0, 0, time.UTC)
|
||||
store.Append(ocds.Release{OCID: "old", ID: "000001-2026", Date: t0, Tag: []string{"tender"}})
|
||||
store.Append(ocds.Release{
|
||||
OCID: "old",
|
||||
ID: "000001-2026",
|
||||
Date: t0,
|
||||
Tag: []string{"tender"},
|
||||
})
|
||||
store.Append(
|
||||
ocds.Release{OCID: "new", ID: "000002-2026", Date: t0.Add(time.Hour), Tag: []string{"tender"}},
|
||||
ocds.Release{
|
||||
OCID: "new",
|
||||
ID: "000002-2026",
|
||||
Date: t0.Add(time.Hour),
|
||||
Tag: []string{"tender"},
|
||||
},
|
||||
)
|
||||
|
||||
//nolint:gosec // G404: seeded math/rand keeps the test deterministic
|
||||
|
||||
Reference in new issue
Block a user