Added fetch service and a mock Find a Tender server

Fetching:
- Add `fetch service`, which runs a worker per enabled source until
  SIGINT/SIGTERM (internal/fetcher)
- Add util.Worker: runs a task on an interval with a since-checkpoint
  that only advances on success, and a shutdown timeout for in-flight
  work; plus AwaitTask/AwaitTaskE for calls that ignore cancellation
- Add fts_mock and sleeper sources
- Sleeper: sleeps periodically to emulate work being done. Can be
  instructed to block the shutdown signal or handle it cleanly

Mock FTS server (cmd/mock-fts, internal/mock/findATender):
- Serves GET /api/1.0/ocdsReleasePackages with FTS paging, query
  validation, UK local time handling and per-client rate limiting
- Generates fictional releases from an embedded catalog, moving
  procedures through planning, tender and award, and keeps them in a
  JSON Lines store that survives restarts
- Add logging.AccessLog for combined/common access logs

.gitignore

@@ -38,6 +38,4 @@ build/
 # Release directory
 release/
 
-# Generated code directories
-internal/config/util/
-!internal/config/util/util.go
+mock-data/

cmd/mock-fts/main.go

@@ -0,0 +1,301 @@
+/*
+Copyright © 2026 Craig Williams <craig@avsdev.uk>
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Lesser General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Lesser General Public License for more details.
+
+You should have received a copy of the GNU Lesser General Public License
+along with this program. If not, see <http://www.gnu.org/licenses/>.
+*/
+// Mock of the Find a Tender (FTS) OCDS release package endpoint:
+//
+//	GET /api/1.0/ocdsReleasePackages?updatedFrom=2026-10-06T00:00:00&limit=5
+package main
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"log"
+	"net/http"
+	"os"
+	"os/signal"
+	"path/filepath"
+	"syscall"
+	"time"
+	_ "time/tzdata"
+
+	"git.avsdev.uk/Go/job-search/internal/logging"
+	mock "git.avsdev.uk/Go/job-search/internal/mock/findATender"
+	"github.com/spf13/cobra"
+)
+
+// options holds the values of every command-line flag.
+type options struct {
+	addr          string
+	baseURL       string
+	dataDir       string
+	reset         bool
+	backfill      time.Duration
+	minGap        time.Duration
+	maxGap        time.Duration
+	rateLimit     int
+	rateWindow    time.Duration
+	accessLogPath string
+	logFormat     string
+	tz            string
+}
+
+const longHelp = `Mock of the Find a Tender (FTS) OCDS release package endpoint:
+
+  GET /api/1.0/ocdsReleasePackages?updatedFrom=2026-10-06T00:00:00&limit=5
+
+Query parameters (mirroring the real service):
+
+  updatedFrom  yyyy-MM-ddTHH:mm:ss   releases updated at or after this time
+  updatedTo    yyyy-MM-ddTHH:mm:ss   releases updated at or before this time (optional)
+  limit        1..100 (default 100)  page size
+  cursor       opaque token from links.next
+
+Like the real service, updatedFrom and updatedTo are UK local time
+(Europe/London, so BST in summer); change this with --tz. Values with an
+explicit offset or Z are taken as given. Release dates are stored and
+returned in UTC.
+
+Data: the catalog of buyers, suppliers, tender subjects, procurement methods
+and classifications that releases are generated from is embedded in the
+build from internal/mock/findATender/data/catalog.json. The mock stops with an
+error if it is invalid.
+
+Releases are generated from that catalog and appended to
+<data-dir>/releases.jsonl, so they survive restarts. A background worker
+publishes a new release every --min-gap to --max-gap. On startup the mock
+fills any gap since the last stored release, going back at most --backfill
+on a fresh data directory. Use --reset to discard stored releases.
+
+Rate limiting: each client (by remote IP) may make --rate-limit requests per
+--rate-window. Windows are fixed and anchored on the client's first request
+of the current clock hour. Over the limit the mock returns 429 Too Many
+Requests, a Retry-After header, and the body
+"Rate limit of N exceeded. Please retry after X seconds."
+
+Access logging: every request is written in Apache/nginx "combined" format
+(or "common" with --log-format) to stdout, or to the --access-log file.
+Startup and error messages go to stderr.`
+
+func main() {
+	// Cancelled on Ctrl+C or SIGTERM; Cobra hands it to the command.
+	ctx, stop := signal.NotifyContext(
+		context.Background(), os.Interrupt, syscall.SIGTERM,
+	)
+	defer stop()
+
+	if err := newRootCmd().ExecuteContext(ctx); err != nil {
+		// Cobra has already printed the error.
+		stop()
+		os.Exit(1)
+	}
+}
+
+func newRootCmd() *cobra.Command {
+	o := &options{}
+
+	cmd := &cobra.Command{
+		Use:          "fts-mock",
+		Short:        "Mock of the Find a Tender OCDS release package API",
+		Long:         longHelp,
+		Args:         cobra.NoArgs,
+		SilenceUsage: true,
+		Example: `
+  fts-mock --addr :8080
+  fts-mock --data-dir ./mock-data --reset
+  fts-mock --min-gap 5s --max-gap 10s --rate-limit 1000`,
+		PreRunE: func(cmd *cobra.Command, args []string) error {
+			return o.validate()
+		},
+		RunE: func(cmd *cobra.Command, args []string) error {
+			return run(cmd.Context(), o)
+		},
+	}
+
+	f := cmd.Flags()
+	f.SortFlags = false // show flags in the order defined here
+
+	f.StringVar(&o.addr, "addr", ":8080", "listen address")
+	f.StringVar(
+		&o.baseURL,
+		"base-url",
+		"http://localhost:8080",
+		"public base URL used in uri and links.next",
+	)
+	f.StringVar(
+		&o.dataDir,
+		"data-dir",
+		"./mock-data",
+		"directory where releases.jsonl will be stored",
+	)
+
+	f.BoolVar(
+		&o.reset,
+		"reset",
+		false,
+		"delete stored releases and regenerate the --backfill history",
+	)
+
+	f.DurationVar(
+		&o.backfill,
+		"backfill",
+		72*time.Hour,
+		"history to generate when the data directory is empty",
+	)
+	f.DurationVar(&o.minGap, "min-gap", 30*time.Second, "minimum time between generated releases")
+	f.DurationVar(&o.maxGap, "max-gap", 5*time.Minute, "maximum time between generated releases")
+
+	f.IntVar(&o.rateLimit, "rate-limit", 12, "requests allowed per window per client")
+	f.DurationVar(&o.rateWindow, "rate-window", 5*time.Minute, "rate limit window length")
+
+	f.StringVar(&o.accessLogPath, "access-log", "-", `access log file, or "-" for stdout`)
+	f.StringVar(&o.logFormat, "log-format", "combined", `access log format: "combined" or "common"`)
+	f.StringVar(
+		&o.tz,
+		"tz",
+		"Europe/London",
+		`time zone for updatedFrom/updatedTo without an offset ("Local" for this machine's)`,
+	)
+
+	return cmd
+}
+
+// validate checks flag values before anything is started.
+func (o *options) validate() error {
+	if o.logFormat != "combined" && o.logFormat != "common" {
+		return fmt.Errorf(`--log-format must be "combined" or "common", got %q`, o.logFormat)
+	}
+	if o.minGap <= 0 || o.maxGap < o.minGap {
+		return errors.New("--min-gap must be positive and no larger than --max-gap")
+	}
+	if o.rateLimit < 1 || o.rateWindow <= 0 {
+		return errors.New("--rate-limit must be at least 1 and --rate-window positive")
+	}
+	if _, err := time.LoadLocation(o.tz); err != nil {
+		return fmt.Errorf("--tz: %w", err)
+	}
+	return nil
+}
+
+// run starts the mock and blocks until ctx is cancelled or the server fails.
+func run(ctx context.Context, o *options) error {
+	loc, err := time.LoadLocation(o.tz) // already validated
+	if err != nil {
+		return fmt.Errorf("--tz: %w", err)
+	}
+	if err := os.MkdirAll(o.dataDir, 0o750); err != nil {
+		return err
+	}
+
+	// ----- catalog -----
+	cat, err := mock.LoadCatalog()
+	if err != nil {
+		return err
+	}
+	log.Printf(
+		"catalog: %d buyers, %d suppliers, %d subjects, %d methods, %d classifications",
+		len(
+			cat.Buyers,
+		),
+		len(cat.Suppliers),
+		len(cat.Subjects),
+		len(cat.Methods),
+		len(cat.Classifications),
+	)
+
+	// ----- store -----
+	storePath := filepath.Join(o.dataDir, mock.ReleasesFile)
+	if o.reset {
+		if err := os.Remove(storePath); err != nil && !errors.Is(err, os.ErrNotExist) {
+			return err
+		}
+		log.Print("store: reset")
+	}
+	store, err := mock.OpenStore(storePath)
+	if err != nil {
+		return err
+	}
+	defer store.Close()
+
+	// ----- generator -----
+	gen := mock.NewGenerator(cat, store, time.Now().UnixNano())
+	n, err := gen.CatchUp(time.Now(), o.backfill, o.minGap, o.maxGap)
+	if err != nil {
+		return err
+	}
+
+	log.Printf("store: %d releases (%d generated to catch up)", store.Len(), n)
+
+	// The generator and server share a context that also ends if the server
+	// fails, so the generator is never left running after run returns.
+	ctx, cancel := context.WithCancel(ctx)
+	defer cancel()
+
+	workerDone := make(chan struct{})
+	go func() {
+		defer close(workerDone)
+		gen.Run(ctx, o.minGap, o.maxGap)
+	}()
+
+	// ----- HTTP -----
+	out := os.Stdout
+	if o.accessLogPath != "-" {
+		f, err := os.OpenFile(
+			o.accessLogPath, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0o600,
+		)
+		if err != nil {
+			cancel()
+			<-workerDone
+			return fmt.Errorf("opening access log: %w", err)
+		}
+		defer f.Close()
+		out = f
+	}
+	accessLogger := log.New(out, "", 0)
+
+	rl := mock.NewRateLimiter(o.rateLimit, o.rateWindow)
+	mux := http.NewServeMux()
+	mux.HandleFunc(
+		mock.BasePath, rl.Middleware(mock.NewHandler(store, o.baseURL, loc)),
+	)
+
+	httpSrv := &http.Server{
+		Addr:              o.addr,
+		Handler:           logging.AccessLog(accessLogger, o.logFormat == "combined", mux),
+		ReadHeaderTimeout: 10 * time.Second,
+	}
+	go func() {
+		<-ctx.Done()
+		shutdownCtx, cancelShutdown := context.WithTimeout(
+			context.Background(), 5*time.Second,
+		)
+		defer cancelShutdown()
+		httpSrv.Shutdown(shutdownCtx)
+	}()
+
+	log.Printf("FTS mock listening on %s%s (query times in %s)", o.addr, mock.BasePath, loc)
+	serveErr := httpSrv.ListenAndServe()
+
+	cancel() // stop the worker (and the shutdown goroutine) if the server failed
+	<-workerDone
+
+	if serveErr != nil && !errors.Is(serveErr, http.ErrServerClosed) {
+		return serveErr
+	}
+	log.Print("stopped")
+
+	return nil
+}

internal/cli/fetch/command.go

@@ -19,5 +19,6 @@ func NewCommand(opts *cli.Options) *cobra.Command {
 		},
 	}
 	cmd.AddCommand(commandOnce(opts))
+	cmd.AddCommand(commandService(opts))
 	return cmd
 }

internal/cli/fetch/service.go

@@ -0,0 +1,70 @@
+package fetch
+
+import (
+	"context"
+	"fmt"
+	"log/slog"
+	"os"
+	"os/signal"
+	"syscall"
+	"time"
+
+	"git.avsdev.uk/Go/job-search/internal/cli"
+	"git.avsdev.uk/Go/job-search/internal/fetcher"
+	"github.com/spf13/cobra"
+)
+
+func commandService(opts *cli.Options) *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   "service",
+		Short: "Fetch current opportunities - background service",
+		PersistentPreRun: func(cmd *cobra.Command, args []string) {
+			slog.Debug("fetch.service.PersistentPreRun")
+		},
+		PersistentPostRun: func(cmd *cobra.Command, args []string) {
+			slog.Debug("fetch.service.PersistentPostRun")
+		},
+		RunE: func(cmd *cobra.Command, args []string) error {
+			slog.Debug("fetch.service.Run")
+
+			foreground, _ := cmd.Flags().GetBool("foreground")
+
+			conf, err := opts.RunSetup(cmd, true, foreground)
+			if err != nil {
+				return fmt.Errorf("no configuration loaded (%w)", err)
+			}
+
+			since, sinceErr := cmd.Flags().GetTime("since")
+			if sinceErr != nil {
+				return sinceErr
+			}
+
+			// ctx is cancelled on the first SIGINT (Ctrl+C) or SIGTERM.
+			ctx, stop := signal.NotifyContext(
+				context.Background(), os.Interrupt, syscall.SIGTERM,
+			)
+			defer stop()
+
+			// After the first signal, restore default signal handling so a second
+			// Ctrl+C force-kills the process if graceful shutdown hangs.
+			go func() {
+				<-ctx.Done()
+				stop()
+			}()
+
+			fetcher.Run(ctx, conf, since)
+
+			return nil
+		},
+	}
+	cmd.Flags().Time(
+		"since",
+		time.Time{},
+		[]string{
+			time.DateOnly, time.DateTime,
+		},
+		"contract updates published since YYYY-MM-DD[ hh:mm:ss]",
+	)
+	cmd.Flags().Bool("foreground", false, "run in the foreground")
+	return cmd
+}

internal/config/conf_source_ftsMock.go

@@ -0,0 +1,32 @@
+package config
+
+import "reflect"
+
+const JSCKSourceFTSMock string = "fts_mock"
+
+type JobSearchConfigSourceFTSMock struct {
+	JobSearchConfigSource
+	APIRoot string `cfg:"api_root"`
+}
+
+/*
+ * General section functions
+ * See related methods in section.go
+ */
+func (c JobSearchConfigSourceFTSMock) Map() map[string]any { return mapOf(c) }
+
+func (c JobSearchConfigSourceFTSMock) Flatten() map[string]any {
+	return flattenOf(JSCKSourceFTSMock, c)
+}
+
+func (c JobSearchConfigSourceFTSMock) TypeOf(key string) (reflect.Kind, error) {
+	return typeOf(c, key)
+}
+
+func (c JobSearchConfigSourceFTSMock) ValueOf(key string) (any, error) {
+	return valueOf(c, key)
+}
+
+func (c *JobSearchConfigSourceFTSMock) SetValue(key string, value any) error {
+	return setValue(c, key, value)
+}

internal/config/conf_source_sleeper.go

@@ -0,0 +1,32 @@
+package config
+
+import "reflect"
+
+const JSCKSourceSleeper string = "sleeper"
+
+type JobSearchConfigSourceSleeper struct {
+	JobSearchConfigSource
+	BlockShutdown bool `cfg:"block_shutdown"`
+}
+
+/*
+ * General section functions
+ * See related methods in section.go
+ */
+func (c JobSearchConfigSourceSleeper) Map() map[string]any { return mapOf(c) }
+
+func (c JobSearchConfigSourceSleeper) Flatten() map[string]any {
+	return flattenOf(JSCKSourceSleeper, c)
+}
+
+func (c JobSearchConfigSourceSleeper) TypeOf(key string) (reflect.Kind, error) {
+	return typeOf(c, key)
+}
+
+func (c JobSearchConfigSourceSleeper) ValueOf(key string) (any, error) {
+	return valueOf(c, key)
+}
+
+func (c *JobSearchConfigSourceSleeper) SetValue(key string, value any) error {
+	return setValue(c, key, value)
+}

internal/config/conf_sources.go

@@ -10,6 +10,8 @@ const JSCKSources string = "sources"
 // Each field's cfg tag must match the section's JSCK* constant.
 type JobSearchConfigSources struct {
 	FindATender JobSearchConfigSourceFindATender `cfg:"find_a_tender"`
+	FTSMock     JobSearchConfigSourceFTSMock     `cfg:"fts_mock"`
+	Sleeper     JobSearchConfigSourceSleeper     `cfg:"sleeper"`
 }
 
 /*

internal/fetcher/fetcher.go

@@ -0,0 +1,45 @@
+package fetcher
+
+import (
+	"context"
+	"fmt"
+	"log/slog"
+	"sync"
+	"time"
+
+	"git.avsdev.uk/Go/job-search/internal/config"
+	"git.avsdev.uk/Go/job-search/internal/util"
+)
+
+var ErrNoNewReleases = fmt.Errorf("%w: no new releases", util.ErrWorkerNoWork)
+
+func Run(
+	ctx context.Context, conf *config.JobSearchConfig, since time.Time,
+) {
+	var wg sync.WaitGroup
+
+	start := func(
+		name string,
+		run func(context.Context, *config.JobSearchConfig, time.Time) error,
+	) {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			if err := run(ctx, conf, since); err != nil {
+				slog.Error("source stopped", "source", name, "err", err)
+			}
+		}()
+	}
+
+	if conf.Sources.FindATender.Enabled {
+		start("FindATender", runFindAOpportunity)
+	}
+	if conf.Sources.FTSMock.Enabled {
+		start("FTSMock", runFTSMock)
+	}
+	if conf.Sources.Sleeper.Enabled {
+		start("Sleeper", runSleeper)
+	}
+
+	wg.Wait()
+}

internal/fetcher/worker_findATender.go

@@ -0,0 +1,47 @@
+package fetcher
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"log/slog"
+	"time"
+
+	"git.avsdev.uk/Go/job-search/internal/config"
+	"git.avsdev.uk/Go/job-search/internal/findATender"
+	"git.avsdev.uk/Go/job-search/internal/types"
+	"git.avsdev.uk/Go/job-search/internal/util"
+)
+
+func runFindAOpportunity(
+	ctx context.Context, conf *config.JobSearchConfig, since time.Time,
+) error {
+	w := util.Worker{
+		Name:     "FindAOpportunity",
+		Interval: conf.Sources.FindATender.Interval,
+		Task: func(ctx context.Context, since time.Time) error {
+			return fetchFindAOpportunity(ctx, conf, since)
+		},
+	}
+	return w.Run(ctx, since)
+}
+
+func fetchFindAOpportunity(
+	ctx context.Context, conf *config.JobSearchConfig, since time.Time,
+) error {
+	opportunities, err := util.AwaitTask(ctx, func() ([]*types.Opportunity, error) {
+		ft := findATender.New(conf, conf.Sources.FindATender.APIRoot)
+		defer ft.Shutdown()
+		return ft.With(ctx).FetchAll(since, true)
+	})
+	if errors.Is(err, findATender.ErrNoReleases) {
+		return ErrNoNewReleases
+	}
+	if err != nil {
+		return err
+	}
+
+	slog.Info(fmt.Sprintf("Found %d opportunities", len(opportunities)))
+
+	return nil
+}

internal/fetcher/worker_ftsMock.go

@@ -0,0 +1,47 @@
+package fetcher
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"log/slog"
+	"time"
+
+	"git.avsdev.uk/Go/job-search/internal/config"
+	"git.avsdev.uk/Go/job-search/internal/findATender"
+	"git.avsdev.uk/Go/job-search/internal/types"
+	"git.avsdev.uk/Go/job-search/internal/util"
+)
+
+func runFTSMock(
+	ctx context.Context, conf *config.JobSearchConfig, since time.Time,
+) error {
+	w := util.Worker{
+		Name:     "FTSMock",
+		Interval: conf.Sources.FTSMock.Interval,
+		Task: func(ctx context.Context, since time.Time) error {
+			return fetchFTSMock(ctx, conf, since)
+		},
+	}
+	return w.Run(ctx, since)
+}
+
+func fetchFTSMock(
+	ctx context.Context, conf *config.JobSearchConfig, since time.Time,
+) error {
+	opportunities, err := util.AwaitTask(ctx, func() ([]*types.Opportunity, error) {
+		ft := findATender.New(conf, conf.Sources.FTSMock.APIRoot)
+		defer ft.Shutdown()
+		return ft.With(ctx).FetchAll(since, true)
+	})
+	if errors.Is(err, findATender.ErrNoReleases) {
+		return ErrNoNewReleases
+	}
+	if err != nil {
+		return err
+	}
+
+	slog.Info(fmt.Sprintf("Found %d opportunities", len(opportunities)))
+
+	return nil
+}

internal/fetcher/worker_sleeper.go

@@ -0,0 +1,50 @@
+package fetcher
+
+import (
+	"context"
+	"time"
+
+	"git.avsdev.uk/Go/job-search/internal/config"
+	"git.avsdev.uk/Go/job-search/internal/util"
+)
+
+// sleepDuration is how long the sleeper's simulated work takes.
+const sleepDuration = 60 * time.Second
+
+// Test source that just sleeps for 60 seconds to simulate doing work
+func runSleeper(
+	ctx context.Context, conf *config.JobSearchConfig, since time.Time,
+) error {
+	blockShutdown := conf.Sources.Sleeper.BlockShutdown
+	w := util.Worker{
+		Name:     "Sleeper",
+		Interval: conf.Sources.Sleeper.Interval,
+		Task: func(taskCtx context.Context, since time.Time) error {
+			if blockShutdown {
+				return sleep(taskCtx, true, sleepDuration)
+			} else {
+				return sleep(ctx, false, sleepDuration)
+			}
+		},
+	}
+	return w.Run(ctx, since)
+}
+
+func sleep(
+	ctx context.Context, blockShutdown bool, duration time.Duration,
+) error {
+	return util.AwaitTaskE(ctx, func() error {
+		if blockShutdown {
+			time.Sleep(duration)
+			return nil
+		}
+		timer := time.NewTimer(duration)
+		defer timer.Stop()
+		select {
+		case <-timer.C:
+			return nil
+		case <-ctx.Done():
+			return ctx.Err()
+		}
+	})
+}

internal/logging/accesslog.go

@@ -0,0 +1,100 @@
+package logging
+
+import (
+	"fmt"
+	"log"
+	"net"
+	"net/http"
+	"strconv"
+	"strings"
+	"time"
+)
+
+// ---------- Access logging ----------
+
+// statusRecorder captures the status code and body size for the access log.
+type statusRecorder struct {
+	http.ResponseWriter
+	status int
+	bytes  int
+}
+
+func (r *statusRecorder) WriteHeader(code int) {
+	if r.status == 0 {
+		r.status = code
+	}
+	r.ResponseWriter.WriteHeader(code)
+}
+
+func (r *statusRecorder) Write(b []byte) (int, error) {
+	if r.status == 0 {
+		r.status = http.StatusOK
+	}
+	n, err := r.ResponseWriter.Write(b)
+	r.bytes += n
+	return n, err
+}
+
+// logField escapes a value for a quoted log field the way nginx does
+// (quotes, backslashes and control characters become \xHH) and uses "-"
+// for empty values.
+func logField(s string) string {
+	if s == "" {
+		return "-"
+	}
+	var b strings.Builder
+	for i := 0; i < len(s); i++ {
+		c := s[i]
+		if c == '"' || c == '\\' || c < 0x20 || c == 0x7f {
+			fmt.Fprintf(&b, "\\x%02X", c)
+		} else {
+			b.WriteByte(c)
+		}
+	}
+	return b.String()
+}
+
+// clientKey attempts to produce a client host name as a key identifier to split
+// requests in the log between sources. Could be extended to also check
+// Forwarded-For or other proxy headers
+func clientKey(req *http.Request) string {
+	host, _, err := net.SplitHostPort(req.RemoteAddr)
+	if err != nil {
+		return req.RemoteAddr
+	}
+	return host
+}
+
+func AccessLog(logger *log.Logger, combined bool, next http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
+		start := time.Now()
+		rec := &statusRecorder{ResponseWriter: w}
+		next.ServeHTTP(rec, req)
+		if rec.status == 0 {
+			rec.status = http.StatusOK
+		}
+
+		user := "-"
+		if u, _, ok := req.BasicAuth(); ok && u != "" {
+			user = logField(u)
+		}
+		size := "-" // CLF uses "-" for an empty body
+		if rec.bytes > 0 {
+			size = strconv.Itoa(rec.bytes)
+		}
+		line := fmt.Sprintf(`%s - %s [%s] "%s" %d %s`,
+			clientKey(req),
+			user,
+			start.Format("02/Jan/2006:15:04:05 -0700"),
+			logField(req.Method+" "+req.RequestURI+" "+req.Proto),
+			rec.status,
+			size,
+		)
+		if combined {
+			line += fmt.Sprintf(
+				` "%s" "%s"`, logField(req.Referer()), logField(req.UserAgent()),
+			)
+		}
+		logger.Println(line)
+	})
+}

internal/mock/findATender/catalog.go

@@ -0,0 +1,184 @@
+package mock
+
+import (
+	_ "embed"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"math/rand"
+	"sort"
+	"strings"
+
+	"git.avsdev.uk/Go/job-search/internal/mock/findATender/ocds"
+)
+
+// Method is a procurement method and its details, as published on a tender.
+type Method struct {
+	Method  string `json:"procurementMethod"`
+	Details string `json:"procurementMethodDetails,omitempty"`
+}
+
+// embeddedCatalog is data/catalog.json, compiled into the binary. Rebuild after
+// changing the file.
+//
+//go:embed data/catalog.json
+var embeddedCatalog []byte
+
+// Catalog is the pool the generator draws from. It is read from
+// ./data/catalog.json. Classifications holds every classification the subjects
+// use, keyed "{scheme}-{id}" (e.g. "CPV-90513000"), and subjects and their
+// items refer to entries by that key.
+type Catalog struct {
+	Buyers          []ocds.Organization            `json:"buyers"`
+	Suppliers       []ocds.Organization            `json:"suppliers"`
+	Subjects        []Subject                      `json:"subjects"`
+	Methods         []Method                       `json:"methods"`
+	Classifications map[string]ocds.Classification `json:"classifications"`
+}
+
+// Subject is a tender as stored in the catalog: an ocds.Tender whose main
+// classification and additionalClassifications, on the tender and on each
+// item, are keys into Catalog.Classifications. The outer fields shadow the
+// embedded ones of the same JSON name, so everything else is read as
+// ocds.Tender.
+type Subject struct {
+	ocds.Tender
+	Classification            string        `json:"classification,omitempty"`
+	AdditionalClassifications []string      `json:"additionalClassifications,omitempty"`
+	Items                     []SubjectItem `json:"items,omitempty"`
+}
+
+// SubjectItem is an ocds.Item whose additionalClassifications are keys into
+// Catalog.Classifications.
+type SubjectItem struct {
+	ocds.Item
+	AdditionalClassifications []string `json:"additionalClassifications,omitempty"`
+}
+
+// LoadCatalog parses the embedded catalog and checks it is usable: every
+// pool has at least one entry and every classification key resolves.
+func LoadCatalog() (*Catalog, error) {
+	return parseCatalog(embeddedCatalog, "embedded catalog")
+}
+
+// parseCatalog decodes and validates catalog JSON; name labels any error.
+func parseCatalog(b []byte, name string) (*Catalog, error) {
+	var c Catalog
+	if err := json.Unmarshal(b, &c); err != nil {
+		return nil, fmt.Errorf("decoding %s: %w", name, err)
+	}
+	if err := c.validate(); err != nil {
+		return nil, fmt.Errorf("invalid %s: %w", name, err)
+	}
+	return &c, nil
+}
+
+func (c *Catalog) validate() error {
+	var errs []error
+
+	for _, pool := range []struct {
+		name string
+		n    int
+	}{
+		{"buyers", len(c.Buyers)},
+		{"suppliers", len(c.Suppliers)},
+		{"subjects", len(c.Subjects)},
+		{"methods", len(c.Methods)},
+		{"classifications", len(c.Classifications)},
+	} {
+		if pool.n == 0 {
+			errs = append(errs, fmt.Errorf("no %s", pool.name))
+		}
+	}
+
+	missing := map[string]bool{}
+	check := func(keys ...string) {
+		for _, k := range keys {
+			if _, ok := c.Classifications[k]; k != "" && !ok {
+				missing[k] = true
+			}
+		}
+	}
+
+	for _, s := range c.Subjects {
+		check(s.Classification)
+		check(s.AdditionalClassifications...)
+		for _, it := range s.Items {
+			check(it.AdditionalClassifications...)
+		}
+	}
+
+	if len(missing) > 0 {
+		keys := make([]string, 0, len(missing))
+		for k := range missing {
+			keys = append(keys, k)
+		}
+		sort.Strings(keys)
+		if len(keys) > 5 {
+			keys = append(keys[:5], "...")
+		}
+		errs = append(errs, fmt.Errorf(
+			"%d classification key(s) not found in classifications: %s",
+			len(missing), strings.Join(keys, ", "),
+		))
+	}
+
+	return errors.Join(errs...)
+}
+
+// ---------- picking (always returns a copy the caller may modify) ----------
+
+func (c *Catalog) buyer(r *rand.Rand) ocds.Organization {
+	return clone(c.Buyers[r.Intn(len(c.Buyers))])
+}
+
+func (c *Catalog) supplier(r *rand.Rand) ocds.Organization {
+	return clone(c.Suppliers[r.Intn(len(c.Suppliers))])
+}
+
+func (c *Catalog) method(r *rand.Rand) Method {
+	return c.Methods[r.Intn(len(c.Methods))]
+}
+
+// subject picks a subject and returns it as an ocds.Tender, with its
+// classification keys resolved to full classifications.
+func (c *Catalog) subject(r *rand.Rand) ocds.Tender {
+	s := clone(c.Subjects[r.Intn(len(c.Subjects))])
+	t := s.Tender
+	t.Classification = c.resolveOne(s.Classification)
+	t.AdditionalClassifications = c.resolve(s.AdditionalClassifications)
+	t.Items = nil
+	for _, it := range s.Items {
+		item := it.Item
+		item.AdditionalClassifications = c.resolve(it.AdditionalClassifications)
+		t.Items = append(t.Items, item)
+	}
+	return t
+}
+
+func (c *Catalog) resolveOne(key string) *ocds.Classification {
+	if cl, ok := c.Classifications[key]; ok {
+		return &cl
+	}
+	return nil
+}
+
+func (c *Catalog) resolve(keys []string) []ocds.Classification {
+	var out []ocds.Classification
+	for _, k := range keys {
+		if cl, ok := c.Classifications[k]; ok {
+			out = append(out, cl)
+		}
+	}
+	return out
+}
+
+// clone deep-copies a value through JSON, which is all the generated types need.
+func clone[T any](v T) T {
+	var out T
+	b, err := json.Marshal(v)
+	if err == nil {
+		_ = json.Unmarshal(b, &out)
+	}
+	return out
+}

internal/mock/findATender/doc.go

@@ -0,0 +1,13 @@
+// Package mock implements a mock of the Find a Tender (FTS) OCDS release
+// package endpoint (GET /api/1.0/ocdsReleasePackages): a catalog of buyers,
+// suppliers, subjects and methods loaded from catalog.json, a persistent
+// release store, a generator with a background worker, the HTTP handler and a
+// per-client rate limiter.
+package mock
+
+// Files kept in the mock-data directory.
+//
+//	ReleasesFile is created and appended to by the mock.
+const (
+	ReleasesFile = "releases.jsonl"
+)

internal/mock/findATender/generator.go

@@ -0,0 +1,274 @@
+package mock
+
+import (
+	"context"
+	"fmt"
+	"hash/fnv"
+	"log"
+	"math"
+	"math/rand"
+	"time"
+
+	"git.avsdev.uk/Go/job-search/internal/mock/findATender/ocds"
+)
+
+const ocidPrefix = "ocds-mock-"
+
+// Generator makes new releases from the catalog. About a third of the time it
+// moves an existing procedure on (planning -> tender -> award) so the same ocid
+// appears in several releases, as it does on the real service.
+type Generator struct {
+	cat   *Catalog
+	store *Store
+	rnd   *rand.Rand
+}
+
+func NewGenerator(cat *Catalog, store *Store, seed int64) *Generator {
+	return &Generator{
+		cat:   cat,
+		store: store,
+		rnd:   rand.New(rand.NewSource(seed)),
+	}
+}
+
+func (g *Generator) Make(at time.Time) ocds.Release {
+	at = at.UTC().Truncate(time.Second)
+	if g.rnd.Float64() < 0.35 {
+		if prev, ok := g.store.RandomOpen(g.rnd, at.AddDate(0, 0, -60)); ok {
+			return g.advance(prev, at)
+		}
+	}
+	return g.newProcedure(at)
+}
+
+func (g *Generator) newProcedure(at time.Time) ocds.Release {
+	buyer := g.cat.buyer(g.rnd)
+	ensurePartyID(&buyer)
+	buyer.Roles = []string{"buyer"}
+	m := g.cat.method(g.rnd)
+
+	tag, status := "tender", "active"
+	if g.rnd.Float64() < 0.2 {
+		tag, status = "planning", "planned"
+	}
+	ocid := g.newOCID()
+
+	t := g.cat.subject(g.rnd)
+	t.ID = ocds.ID(ocid)
+	t.ProcurementMethod = m.Method
+	t.ProcurementMethodDetails = m.Details
+
+	factor := 0.5 + g.rnd.Float64()*1.5
+	if t.Value == nil {
+		amount := float64((g.rnd.Intn(5000) + 10) * 1000)
+		t.Value = &ocds.Value{
+			Amount:   &amount,
+			Currency: "GBP",
+		}
+	} else {
+		scaleValue(t.Value, factor)
+	}
+
+	for i := range t.Lots {
+		scaleValue(t.Lots[i].Value, factor)
+	}
+
+	if len(t.Lots) == 0 {
+		v := clone(*t.Value)
+		t.Lots = []ocds.Lot{{
+			ID:    "1",
+			Title: t.Title,
+			Value: &v,
+		}}
+	}
+
+	setStatus(&t, status)
+	if status == "active" {
+		t.TenderPeriod = &ocds.Period{
+			EndDate: ptr(at.AddDate(0, 0, 30+g.rnd.Intn(30))),
+		}
+	}
+
+	return ocds.Release{
+		OCID:           ocid,
+		ID:             g.store.NextNoticeID(at.Year()),
+		Date:           at,
+		Tag:            []string{tag},
+		InitiationType: "tender",
+		Parties:        []ocds.Organization{buyer},
+		Buyer: &ocds.OrganizationReference{
+			ID:   ocds.ID(buyer.ID),
+			Name: buyer.Name,
+		},
+		Tender:   &t,
+		Language: "en",
+	}
+}
+
+func (g *Generator) advance(prev ocds.Release, at time.Time) ocds.Release {
+	r := clone(prev)
+	r.ID = g.store.NextNoticeID(at.Year())
+	r.Date = at
+	r.Awards = nil
+	if r.Tender == nil {
+		r.Tender = &ocds.Tender{
+			ID: ocds.ID(r.OCID),
+		}
+	}
+
+	next := "award"
+	switch prev.Tag[0] {
+	case "planning":
+		next = "tender"
+	case "tender":
+		if g.rnd.Float64() < 0.25 {
+			next = "tenderUpdate"
+		}
+	}
+	r.Tag = []string{next}
+
+	switch next {
+	case "tender":
+		setStatus(r.Tender, "active")
+		r.Tender.TenderPeriod = &ocds.Period{
+			EndDate: ptr(at.AddDate(0, 0, 30+g.rnd.Intn(30))),
+		}
+	case "tenderUpdate":
+		// A typical update: the deadline is extended by two weeks.
+		end := at.AddDate(0, 0, 30)
+		if tp := r.Tender.TenderPeriod; tp != nil && tp.EndDate != nil {
+			end = *tp.EndDate
+		}
+		r.Tender.TenderPeriod = &ocds.Period{
+			EndDate: ptr(end.AddDate(0, 0, 14)),
+		}
+	case "award":
+		setStatus(r.Tender, "complete")
+		sup := g.cat.supplier(g.rnd)
+		ensurePartyID(&sup)
+		sup.Roles = []string{"supplier"}
+		r.Parties = append(r.Parties, sup)
+
+		var value *ocds.Value
+		if r.Tender.Value != nil {
+			v := clone(*r.Tender.Value)
+			scaleValue(&v, 0.8+g.rnd.Float64()*0.2)
+			value = &v
+		}
+
+		var lotIDs []string
+		for _, lot := range r.Tender.Lots {
+			if lot.ID != "" {
+				lotIDs = append(lotIDs, lot.ID)
+			}
+		}
+
+		r.Awards = []ocds.Award{{
+			ID:     "1",
+			Status: "active",
+			Date:   ptr(at),
+			Value:  value,
+			Suppliers: []ocds.OrganizationReference{
+				{
+					ID:   ocds.ID(sup.ID),
+					Name: sup.Name,
+				},
+			},
+			RelatedLots: lotIDs,
+		}}
+	}
+
+	return r
+}
+
+func (g *Generator) newOCID() string {
+	for {
+		id := fmt.Sprintf("%s%06x", ocidPrefix, g.rnd.Intn(1<<24))
+		if !g.store.HasOCID(id) {
+			return id
+		}
+	}
+}
+
+func (g *Generator) gap(min, max time.Duration) time.Duration {
+	if max <= min {
+		return min
+	}
+	return min + time.Duration(g.rnd.Int63n(int64(max-min)+1))
+}
+
+// CatchUp fills the period since the last stored release (or the last backfill
+// period, whichever is shorter) so restarts leave no gap in the timeline.
+func (g *Generator) CatchUp(
+	now time.Time, backfill, min, max time.Duration,
+) (int, error) {
+	start := now.Add(-backfill)
+	if last, ok := g.store.Last(); ok && last.After(start) {
+		start = last
+	}
+
+	n := 0
+	for t := start.Add(g.gap(min, max)); !t.After(now); t = t.Add(g.gap(min, max)) {
+		if err := g.store.AppendNoSync(g.Make(t)); err != nil {
+			return n, err
+		}
+		n++
+	}
+
+	return n, g.store.Sync()
+}
+
+// Run publishes a new release after a random gap between min and max until ctx ends.
+func (g *Generator) Run(ctx context.Context, min, max time.Duration) {
+	for {
+		d := g.gap(min, max)
+		log.Printf("generator: next release in %s", d.Round(time.Second))
+
+		timer := time.NewTimer(d)
+		select {
+		case <-ctx.Done():
+			timer.Stop()
+			return
+		case <-timer.C:
+		}
+
+		r := g.Make(time.Now())
+		if err := g.store.Append(r); err != nil {
+			log.Printf("generator: storing release: %v", err)
+			continue
+		}
+
+		log.Printf("generator: published %s (%s) %s", r.ID, r.Tag[0], r.OCID)
+	}
+}
+
+// ---------- helpers ----------
+
+func ptr[T any](v T) *T { return &v }
+
+func setStatus(t *ocds.Tender, status string) {
+	t.Status = status
+	for i := range t.Lots {
+		t.Lots[i].Status = status
+	}
+}
+
+func scaleValue(v *ocds.Value, f float64) {
+	if v == nil {
+		return
+	}
+	for _, a := range []**float64{&v.Amount, &v.AmountGross} {
+		if *a != nil {
+			*a = ptr(math.Round(**a*f/100) * 100)
+		}
+	}
+}
+
+func ensurePartyID(p *ocds.Organization) {
+	if p.ID != "" {
+		return
+	}
+	h := fnv.New32a()
+	h.Write([]byte(p.Name))
+	p.ID = fmt.Sprintf("GB-FTS-%d", h.Sum32()%1000000)
+}

internal/mock/findATender/handler.go

@@ -0,0 +1,189 @@
+package mock
+
+import (
+	"encoding/base64"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"log"
+	"net/http"
+	"net/url"
+	"strconv"
+	"strings"
+	"time"
+
+	"git.avsdev.uk/Go/job-search/internal/mock/findATender/ocds"
+)
+
+const (
+	defaultLimit = 100
+	maxLimit     = 100
+	timeLayout   = "2006-01-02T15:04:05"
+	BasePath     = "/api/1.0/ocdsReleasePackages"
+)
+
+type server struct {
+	store   *Store
+	baseURL string
+	loc     *time.Location // zone for updatedFrom/updatedTo without an offset; nil means UTC
+}
+
+// NewHandler serves release packages from store. baseURL is the public base
+// URL used in each package's uri and links.next. loc is the time zone that
+// updatedFrom and updatedTo are read in when they carry no offset, as the
+// documented yyyy-MM-ddTHH:mm:ss format doesn't (Europe/London for FTS).
+// Stored release dates are UTC; comparisons are on absolute instants.
+func NewHandler(
+	store *Store, baseURL string, loc *time.Location,
+) http.HandlerFunc {
+	return (&server{store: store, baseURL: baseURL, loc: loc}).handle
+}
+
+func encodeCursor(i int) string {
+	return base64.RawURLEncoding.EncodeToString([]byte("idx:" + strconv.Itoa(i)))
+}
+
+func decodeCursor(c string) (int, error) {
+	b, err := base64.RawURLEncoding.DecodeString(c)
+	if err != nil || !strings.HasPrefix(string(b), "idx:") {
+		return 0, fmt.Errorf("invalid cursor")
+	}
+
+	n, err := strconv.Atoi(strings.TrimPrefix(string(b), "idx:"))
+	if err != nil || n < 0 {
+		return 0, fmt.Errorf("invalid cursor")
+	}
+
+	return n, nil
+}
+
+func badRequest(w http.ResponseWriter, msg string) {
+	w.Header().Set("Content-Type", "application/json")
+	w.WriteHeader(http.StatusBadRequest)
+	json.NewEncoder(w).Encode(map[string]string{"error": msg})
+}
+
+// parseTime reads a query timestamp. Values without an offset (the documented
+// yyyy-MM-ddTHH:mm:ss, or a plain date) are local time in loc; values with an
+// offset or Z (RFC 3339) mean exactly what they say.
+func parseTime(v string, loc *time.Location) (time.Time, error) {
+	if loc == nil {
+		loc = time.UTC
+	}
+
+	if t, err := time.Parse(time.RFC3339, v); err == nil {
+		return t, nil
+	}
+
+	for _, layout := range []string{timeLayout, "2006-01-02"} {
+		if t, err := time.ParseInLocation(layout, v, loc); err == nil {
+			return t, nil
+		}
+	}
+
+	return time.Time{}, fmt.Errorf("must be in format yyyy-MM-ddTHH:mm:ss")
+}
+
+// pageQuery is a parsed release package request.
+type pageQuery struct {
+	from, upper time.Time // releases dated in [from, upper]
+	start       int       // index of the first release to return
+	limit       int
+}
+
+func (s *server) handle(w http.ResponseWriter, req *http.Request) {
+	if req.Method != http.MethodGet {
+		w.WriteHeader(http.StatusMethodNotAllowed)
+		return
+	}
+
+	q := req.URL.Query()
+	pq, err := s.parseQuery(q)
+	if err != nil {
+		badRequest(w, err.Error())
+		return
+	}
+
+	releases, next, more := s.store.Page(pq.from, pq.upper, pq.start, pq.limit)
+	pkg := ocds.ReleasePackage{
+		URI:           s.baseURL + req.URL.RequestURI(),
+		Version:       "1.1",
+		Extensions:    ocds.DeclaredExtensions,
+		PublishedDate: ptr(time.Now().UTC().Truncate(time.Second)),
+		Publisher: &ocds.ReleasePackagePublisher{
+			Name: "OCDS Mock",
+			URI:  s.baseURL,
+		},
+		Releases: releases,
+	}
+	if more {
+		pkg.Links = &ocds.ReleasePackageLinks{Next: s.nextLink(q, next)}
+	}
+
+	w.Header().Set("Content-Type", "application/json; charset=utf-8")
+	w.Header().Set("Access-Control-Allow-Origin", "*")
+	enc := json.NewEncoder(w)
+	enc.SetIndent("", "  ")
+	enc.SetEscapeHTML(false)
+	if err := enc.Encode(pkg); err != nil {
+		log.Printf("handler: writing response: %v", err)
+	}
+}
+
+// parseQuery reads and checks the query parameters. Its errors are meant for
+// the client.
+func (s *server) parseQuery(q url.Values) (pageQuery, error) {
+	pq := pageQuery{limit: defaultLimit, upper: time.Now().UTC()}
+
+	if v := q.Get("limit"); v != "" {
+		n, err := strconv.Atoi(v)
+		if err != nil || n < 1 || n > maxLimit {
+			return pq, fmt.Errorf("limit must be an integer between 1 and %d", maxLimit)
+		}
+		pq.limit = n
+	}
+
+	if v := q.Get("updatedFrom"); v != "" {
+		t, err := parseTime(v, s.loc)
+		if err != nil {
+			return pq, fmt.Errorf("updatedFrom %w", err)
+		}
+		pq.from = t
+	}
+
+	if v := q.Get("updatedTo"); v != "" {
+		t, err := parseTime(v, s.loc)
+		if err != nil {
+			return pq, fmt.Errorf("updatedTo %w", err)
+		}
+		if q.Get("updatedFrom") != "" && t.Before(pq.from) {
+			return pq, errors.New("updatedTo must not be before updatedFrom")
+		}
+		if t.Before(pq.upper) {
+			pq.upper = t
+		}
+	}
+
+	if c := q.Get("cursor"); c != "" {
+		n, err := decodeCursor(c)
+		if err != nil {
+			return pq, err
+		}
+		pq.start = n
+	}
+
+	return pq, nil
+}
+
+// nextLink is the URL of the page starting at release index next: the same
+// query with a new cursor.
+func (s *server) nextLink(q url.Values, next int) string {
+	nq := url.Values{}
+	for k, vs := range q {
+		if k != "cursor" {
+			nq[k] = vs
+		}
+	}
+	nq.Set("cursor", encodeCursor(next))
+	return s.baseURL + BasePath + "?" + nq.Encode()
+}

internal/mock/findATender/ratelimit.go

@@ -0,0 +1,119 @@
+package mock
+
+import (
+	"fmt"
+	"net"
+	"net/http"
+	"strconv"
+	"sync"
+	"time"
+)
+
+// ---------- rate limiting ----------
+
+type clientWindow struct {
+	hour        time.Time // clock hour the anchor belongs to
+	anchor      time.Time // first request of that hour
+	windowStart time.Time // start of the current window
+	count       int       // accepted requests in the current window
+}
+
+type RateLimiter struct {
+	mu      sync.Mutex
+	limit   int
+	window  time.Duration
+	clients map[string]*clientWindow
+	now     func() time.Time
+}
+
+func NewRateLimiter(limit int, window time.Duration) *RateLimiter {
+	return &RateLimiter{
+		limit:   limit,
+		window:  window,
+		clients: map[string]*clientWindow{},
+		now:     time.Now,
+	}
+}
+
+// allow records a request for key and reports whether it may proceed. When it
+// may not, it also returns the whole seconds until the client's next window.
+func (rl *RateLimiter) allow(key string) (bool, int) {
+	rl.mu.Lock()
+	defer rl.mu.Unlock()
+
+	now := rl.now().UTC()
+	hour := now.Truncate(time.Hour)
+
+	c, ok := rl.clients[key]
+	if !ok || !c.hour.Equal(hour) {
+		// First request of this clock hour: it anchors the windows.
+		if !ok && len(rl.clients) > 10000 {
+			rl.prune(hour)
+		}
+		c = &clientWindow{hour: hour, anchor: now, windowStart: now}
+		rl.clients[key] = c
+	}
+
+	// Move to whichever fixed window, counted from the anchor, now falls in.
+	idx := now.Sub(c.anchor) / rl.window
+	if ws := c.anchor.Add(idx * rl.window); !ws.Equal(c.windowStart) {
+		c.windowStart = ws
+		c.count = 0
+	}
+
+	if c.count < rl.limit {
+		c.count++
+		return true, 0
+	}
+
+	// The next window opens at the end of this one, or at the top of the
+	// next hour if that comes first (because a new hour re-anchors).
+	next := c.windowStart.Add(rl.window)
+	if nh := hour.Add(time.Hour); nh.Before(next) {
+		next = nh
+	}
+	secs := int((next.Sub(now) + time.Second - 1) / time.Second) // round up
+	if secs < 1 {
+		secs = 1
+	}
+	return false, secs
+}
+
+// prune drops clients whose state belongs to an earlier hour.
+func (rl *RateLimiter) prune(hour time.Time) {
+	for k, c := range rl.clients {
+		if c.hour.Before(hour) {
+			delete(rl.clients, k)
+		}
+	}
+}
+
+// clientKey produces an identifier for the client based on remote address.
+// Could be updated to support proxy and forwarded-for headers.
+func clientKey(req *http.Request) string {
+	host, _, err := net.SplitHostPort(req.RemoteAddr)
+	if err != nil {
+		return req.RemoteAddr
+	}
+	return host
+}
+
+// Middleware attaches to http handler. Checks to see if a client is allowed
+// to make a request in their current window or returns a "TooManyRequests"
+// status and the time the client should wait before making another request.
+func (rl *RateLimiter) Middleware(next http.HandlerFunc) http.HandlerFunc {
+	return func(w http.ResponseWriter, req *http.Request) {
+		ok, retry := rl.allow(clientKey(req))
+		if !ok {
+			w.Header().Set("Retry-After", strconv.Itoa(retry))
+			w.Header().Set("Content-Type", "text/plain; charset=utf-8")
+			w.WriteHeader(http.StatusTooManyRequests)
+			fmt.Fprintf(
+				w, "Rate limit of %d exceeded. Please retry after %d seconds.",
+				rl.limit, retry,
+			)
+			return
+		}
+		next(w, req)
+	}
+}

internal/mock/findATender/store.go

@@ -0,0 +1,257 @@
+package mock
+
+import (
+	"bufio"
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"log"
+	"math/rand"
+	"os"
+	"path/filepath"
+	"sort"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+
+	"git.avsdev.uk/Go/job-search/internal/mock/findATender/ocds"
+)
+
+// Store keeps every release in memory and appends each new one to a JSON Lines
+// file, so releases survive restarts. Releases are kept in date order.
+type Store struct {
+	mu       sync.RWMutex
+	f        *os.File
+	releases []ocds.Release
+	times    []time.Time
+	latest   map[string]int // ocid -> index of its most recent release
+	lastNum  map[int]int    // year -> highest notice number used
+
+	// Procedures not yet awarded, as a slice plus positions so that adding,
+	// removing and picking one at random are all constant time.
+	open    []string
+	openPos map[string]int
+}
+
+func OpenStore(path string) (*Store, error) {
+	s := &Store{latest: map[string]int{}, lastNum: map[int]int{}, openPos: map[string]int{}}
+	if err := os.MkdirAll(filepath.Dir(path), 0o750); err != nil {
+		return nil, err
+	}
+
+	releases, err := readReleases(path)
+	if err != nil {
+		return nil, err
+	}
+	// The file is written in order, but sort anyway in case it was hand-edited.
+	sort.SliceStable(releases, func(i, j int) bool {
+		return releases[i].Date.Before(releases[j].Date)
+	})
+	for _, r := range releases {
+		s.releases = append(s.releases, r)
+		s.times = append(s.times, r.Date)
+	}
+	for i, r := range s.releases {
+		s.index(i, r)
+	}
+
+	if s.f, err = openForAppend(path); err != nil {
+		return nil, err
+	}
+	return s, nil
+}
+
+// readReleases reads the releases stored at path, skipping lines that are
+// blank or can't be decoded. A missing file holds no releases.
+func readReleases(path string) ([]ocds.Release, error) {
+	in, err := os.Open(path)
+	if os.IsNotExist(err) {
+		return nil, nil
+	}
+	if err != nil {
+		return nil, err
+	}
+	defer in.Close()
+
+	var releases []ocds.Release
+	sc := bufio.NewScanner(in)
+	sc.Buffer(make([]byte, 1<<20), 64<<20)
+	for line := 1; sc.Scan(); line++ {
+		b := bytes.TrimSpace(sc.Bytes())
+		if len(b) == 0 {
+			continue
+		}
+		var r ocds.Release
+		if err := json.Unmarshal(b, &r); err != nil {
+			log.Printf("store: skipping unreadable line %d: %v", line, err)
+			continue
+		}
+		releases = append(releases, r)
+	}
+	if err := sc.Err(); err != nil {
+		return nil, fmt.Errorf("reading %s: %w", path, err)
+	}
+	return releases, nil
+}
+
+// openForAppend opens path for appending releases. If a previous run died
+// mid-write, it ends the partial record so the next one starts on a fresh
+// line.
+func openForAppend(path string) (*os.File, error) {
+	f, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR|os.O_APPEND, 0o600)
+	if err != nil {
+		return nil, err
+	}
+	if st, err := f.Stat(); err == nil && st.Size() > 0 {
+		last := make([]byte, 1)
+		if _, err := f.ReadAt(last, st.Size()-1); err == nil && last[0] != '\n' {
+			if _, err := f.Write([]byte("\n")); err != nil {
+				f.Close()
+				return nil, fmt.Errorf("repairing %s: %w", path, err)
+			}
+		}
+	}
+	return f, nil
+}
+
+func (s *Store) index(i int, r ocds.Release) {
+	s.latest[r.OCID] = i
+	if len(r.Tag) > 0 && r.Tag[0] == "award" {
+		s.closeProcedure(r.OCID)
+	} else if _, ok := s.openPos[r.OCID]; !ok {
+		s.openPos[r.OCID] = len(s.open)
+		s.open = append(s.open, r.OCID)
+	}
+	if num, year, ok := strings.Cut(r.ID, "-"); ok {
+		n, err1 := strconv.Atoi(num)
+		y, err2 := strconv.Atoi(year)
+		if err1 == nil && err2 == nil && n > s.lastNum[y] {
+			s.lastNum[y] = n
+		}
+	}
+}
+
+func (s *Store) closeProcedure(ocid string) {
+	pos, ok := s.openPos[ocid]
+	if !ok {
+		return
+	}
+	last := len(s.open) - 1
+	s.open[pos] = s.open[last]
+	s.openPos[s.open[pos]] = pos
+	s.open = s.open[:last]
+	delete(s.openPos, ocid)
+}
+
+// Append persists a release and flushes it to disk.
+func (s *Store) Append(r ocds.Release) error { return s.append(r, true) }
+
+// AppendNoSync persists a release without an fsync; call Sync afterwards.
+// Used for bulk catch-up, where a sync per record would be very slow.
+func (s *Store) AppendNoSync(r ocds.Release) error { return s.append(r, false) }
+
+func (s *Store) Sync() error {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	return s.f.Sync()
+}
+
+// A date earlier than the newest stored release (for example after a clock
+// change) is moved up so the order stays valid.
+func (s *Store) append(r ocds.Release, sync bool) error {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+
+	r.Date = r.Date.UTC()
+	if n := len(s.times); n > 0 && r.Date.Before(s.times[n-1]) {
+		r.Date = s.times[n-1]
+	}
+	b, err := json.Marshal(r)
+	if err != nil {
+		return err
+	}
+	if _, err := s.f.Write(append(b, '\n')); err != nil {
+		return err
+	}
+	if sync {
+		if err := s.f.Sync(); err != nil {
+			return err
+		}
+	}
+	s.releases = append(s.releases, r)
+	s.times = append(s.times, r.Date)
+	s.index(len(s.releases)-1, r)
+	return nil
+}
+
+// Page returns up to limit releases dated in [from, upper], starting no earlier
+// than position start, plus the position to continue from and whether more exist.
+func (s *Store) Page(from, upper time.Time, start, limit int) ([]ocds.Release, int, bool) {
+	s.mu.RLock()
+	defer s.mu.RUnlock()
+
+	i := sort.Search(len(s.times), func(i int) bool { return !s.times[i].Before(from) })
+	if start > i {
+		i = start
+	}
+	out := make([]ocds.Release, 0, limit)
+	for i < len(s.releases) && len(out) < limit && !s.times[i].After(upper) {
+		out = append(out, s.releases[i])
+		i++
+	}
+	more := i < len(s.releases) && !s.times[i].After(upper)
+	return out, i, more
+}
+
+func (s *Store) Last() (time.Time, bool) {
+	s.mu.RLock()
+	defer s.mu.RUnlock()
+	if len(s.times) == 0 {
+		return time.Time{}, false
+	}
+	return s.times[len(s.times)-1], true
+}
+
+func (s *Store) Len() int {
+	s.mu.RLock()
+	defer s.mu.RUnlock()
+	return len(s.releases)
+}
+
+func (s *Store) HasOCID(ocid string) bool {
+	s.mu.RLock()
+	defer s.mu.RUnlock()
+	_, ok := s.latest[ocid]
+	return ok
+}
+
+// NextNoticeID returns the next FTS-style notice id ("000123-2026") for year.
+func (s *Store) NextNoticeID(year int) string {
+	s.mu.RLock()
+	defer s.mu.RUnlock()
+	return fmt.Sprintf("%06d-%d", s.lastNum[year]+1, year)
+}
+
+// RandomOpen picks a procedure that has not been awarded and whose latest
+// release is after since. Procedures idle since before then are dropped from
+// the open set, as if they had been abandoned.
+func (s *Store) RandomOpen(r *rand.Rand, since time.Time) (ocds.Release, bool) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	for len(s.open) > 0 {
+		ocid := s.open[r.Intn(len(s.open))]
+		i := s.latest[ocid]
+		if s.times[i].After(since) {
+			return s.releases[i], true
+		}
+		s.closeProcedure(ocid)
+	}
+	return ocds.Release{}, false
+}
+
+func (s *Store) Close() error {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	return s.f.Close()
+}

internal/mock/findATender/testdata/catalog.json

@@ -0,0 +1,38 @@
+{
+  "buyers": [
+    {
+      "id": "ZZ-MPON-PAAA-1111-AAAA",
+      "name": "Example Borough Council",
+      "identifier": {"scheme": "ZZ-MPON", "id": "PAAA-1111-AAAA"},
+      "address": {"locality": "Exampleton", "postalCode": "ZZ1 1AA", "countryName": "United Kingdom", "region": "UKJ"},
+      "details": {"classifications": [{"scheme": "UK_CA_TYPE", "id": "publicAuthoritySubCentralGovernment", "description": "Public authority - sub-central government"}]}
+    },
+    {"id": "ZZ-MPON-PBBB-2222-BBBB", "name": "Example NHS Trust"}
+  ],
+  "suppliers": [
+    {"id": "ZZ-MCOH-01234567", "name": "Example Supplies Ltd", "identifier": {"scheme": "ZZ-MCOH", "id": "01234567"}}
+  ],
+  "subjects": [
+    {
+      "title": "Grounds maintenance services",
+      "mainProcurementCategory": "services",
+      "classification": "CPV-77314000",
+      "items": [{"id": "1", "additionalClassifications": ["CPV-77310000"], "deliveryAddresses": [{"region": "UKJ"}], "relatedLot": "1"}],
+      "lots": [{"id": "1", "title": "Lot 1: Parks", "value": {"amountGross": 120000, "amount": 100000, "currency": "GBP"}}],
+      "value": {"amountGross": 120000, "amount": 100000, "currency": "GBP"}
+    },
+    {
+      "title": "Patient transport",
+      "classification": "CPV-60130000"
+    }
+  ],
+  "methods": [
+    {"procurementMethod": "open", "procurementMethodDetails": "Open procedure"},
+    {"procurementMethod": "selective", "procurementMethodDetails": "Competitive flexible procedure"}
+  ],
+  "classifications": {
+    "CPV-60130000": {"scheme": "CPV", "id": "60130000", "description": "Special-purpose road passenger-transport services"},
+    "CPV-77310000": {"scheme": "CPV", "id": "77310000", "description": "Planting and maintenance services of green areas"},
+    "CPV-77314000": {"scheme": "CPV", "id": "77314000", "description": "Grounds maintenance services"}
+  }
+}

internal/util/worker.go

@@ -0,0 +1,176 @@
+package util
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"log/slog"
+	"time"
+)
+
+// DefaultWorkerShutdownTimeout is used when Worker.ShutdownTimeout is zero.
+const DefaultWorkerShutdownTimeout = 10 * time.Second
+
+// ErrWorkerNoWork tells the Worker that a run succeeded but found nothing to do.
+// Tasks return it, or an error wrapping it, to have the run treated as a
+// success (checkpoint advances) while being logged distinctly. For example:
+//
+//	var ErrNoNewReleases = fmt.Errorf("%w: no new releases", ErrWorkerNoWork)
+var ErrWorkerNoWork = errors.New("no work")
+
+// Task processes everything that has changed since the given checkpoint.
+//
+// Returning nil or an error matching ErrNoWork advances the checkpoint to the
+// time this run started. Any other error leaves the checkpoint unchanged, so
+// the next run retries from the same point.
+type Task func(ctx context.Context, since time.Time) error
+
+// Worker runs a Task periodically until its context is cancelled.
+type Worker struct {
+	// Name is attached to every log line as "source".
+	Name string
+	// Interval is how often Task runs. Must be greater than zero.
+	Interval time.Duration
+	// ShutdownTimeout is how long an in-flight Task may keep running after
+	// the parent context is cancelled. Zero means DefaultShutdownTimeout.
+	ShutdownTimeout time.Duration
+	// Task is the work to perform on each tick.
+	Task Task
+}
+
+// Run executes Task immediately, then once per Interval, until ctx is
+// cancelled. It blocks until shutdown has finished.
+func (w Worker) Run(ctx context.Context, since time.Time) error {
+	if w.Task == nil {
+		return fmt.Errorf("worker %q: no task configured", w.Name)
+	}
+	if w.Interval <= 0 {
+		return fmt.Errorf(
+			"worker %q: interval must be greater than 0, got %s",
+			w.Name, w.Interval,
+		)
+	}
+	timeout := w.ShutdownTimeout
+	if timeout <= 0 {
+		timeout = DefaultWorkerShutdownTimeout
+	}
+
+	logger := slog.With("source", w.Name)
+	logger.Info("starting worker", "since", since, "interval", w.Interval)
+
+	// workCtx is NOT cancelled immediately when ctx is. It gives an in-flight
+	// task up to `timeout` to finish before it's cancelled.
+	workCtx, cancelWork := context.WithCancel(context.WithoutCancel(ctx))
+	defer cancelWork()
+	go func() {
+		select {
+		case <-ctx.Done():
+		case <-workCtx.Done():
+			return
+		}
+		select {
+		case <-time.After(timeout):
+			logger.Error("shutdown timeout exceeded, cancelling in-flight task")
+			cancelWork()
+		case <-workCtx.Done():
+		}
+	}()
+
+	ticker := time.NewTicker(w.Interval)
+	defer ticker.Stop()
+
+	if ctx.Err() == nil {
+		since = w.execute(workCtx, logger, since)
+	}
+
+	for {
+		// Check for shutdown first so we don't start new work after a
+		// signal even if a tick is also ready.
+		if ctx.Err() != nil {
+			logger.Warn("shutdown signal received, exiting")
+			return nil
+		}
+
+		select {
+		case <-ctx.Done():
+			logger.Warn("shutdown signal received, exiting")
+			return nil
+		case <-ticker.C:
+			since = w.execute(workCtx, logger, since)
+		}
+	}
+}
+
+// execute runs one iteration of the task, logs the outcome and returns the
+// checkpoint to use for the next run.
+func (w Worker) execute(
+	ctx context.Context, logger *slog.Logger, since time.Time,
+) time.Time {
+	start := time.Now()
+	logger.Info("running task", "since", since)
+
+	err := w.Task(ctx, since)
+	elapsed := time.Since(start)
+
+	switch {
+	case err == nil:
+		logger.Info("task completed", "elapsed", elapsed)
+		return start
+	case errors.Is(err, ErrWorkerNoWork):
+		logger.Info("task completed with nothing to do", "reason", err, "elapsed", elapsed)
+		return start
+	case errors.Is(err, context.Canceled):
+		logger.Warn("task cancelled", "elapsed", elapsed)
+		return since
+	default:
+		logger.Error("task failed", "err", err, "elapsed", elapsed)
+		return since
+	}
+}
+
+// AwaitTask runs fn in its own goroutine and returns its result, or ctx.Err()
+// if ctx is cancelled first. This lets a Task stop waiting on a call that does
+// not respond promptly to cancellation.
+//
+// The result channel is buffered, so if ctx wins the race the goroutine can
+// still deliver its result and exit instead of blocking forever.
+func AwaitTask[T any](ctx context.Context, fn func() (T, error)) (T, error) {
+	type result struct {
+		val T
+		err error
+	}
+	ch := make(chan result, 1)
+	go func() {
+		val, err := fn()
+		ch <- result{val, err}
+	}()
+
+	select {
+	case r := <-ch:
+		return r.val, r.err
+	case <-ctx.Done():
+		var zero T
+		return zero, ctx.Err()
+	}
+}
+
+// AwaitTask runs fn in its own goroutine and returns its result, or ctx.Err()
+// if ctx is cancelled first. This lets a Task stop waiting on a call that does
+// not respond promptly to cancellation.
+//
+// The result channel is buffered, so if ctx wins the race the goroutine can
+// still deliver its result and exit instead of blocking forever.
+func AwaitTaskE(ctx context.Context, fn func() error) error {
+	ch := make(chan error, 1)
+	go func() {
+		err := fn()
+		ch <- err
+	}()
+
+	select {
+	case err := <-ch:
+		return err
+	case <-ctx.Done():
+		return ctx.Err()
+	}
+}