Go/job-search
1
0 Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases 2 Wiki Activity

Added fetch service and a mock Find a Tender server

Browse Source 
Fetching:
- Add `fetch service`, which runs a worker per enabled source until
  SIGINT/SIGTERM (internal/fetcher)
- Add util.Worker: runs a task on an interval with a since-checkpoint
  that only advances on success, and a shutdown timeout for in-flight
  work; plus AwaitTask/AwaitTaskE for calls that ignore cancellation
- Add fts_mock and sleeper sources
- Sleeper: sleeps periodically to emulate work being done. Can be
  instructed to block the shutdown signal or handle it cleanly

Mock FTS server (cmd/mock-fts, internal/mock/findATender):
- Serves GET /api/1.0/ocdsReleasePackages with FTS paging, query
  validation, UK local time handling and per-client rate limiting
- Generates fictional releases from an embedded catalog, moving
  procedures through planning, tender and award, and keeps them in a
  JSON Lines store that survives restarts
- Add logging.AccessLog for combined/common access logs
This commit is contained in:
bwilliams committed 2026-10-09 18:02:32 +01:00
1 parent dcb714dc19
commit 5cb3601e1f
23 files changed
+97717 -3
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

cmd/mock-fts/main.go
+301
View File
@@ -0,0 +1,301 @@
/*
Copyright © 2026 Craig Williams <craig@avsdev.uk>
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
// Mock of the Find a Tender (FTS) OCDS release package endpoint:
//
// GET /api/1.0/ocdsReleasePackages?updatedFrom=2026-10-06T00:00:00&limit=5
package main
import (
"context"
"errors"
"fmt"
"log"
"net/http"
"os"
"os/signal"
"path/filepath"
"syscall"
"time"
_ "time/tzdata"
"git.avsdev.uk/Go/job-search/internal/logging"
mock "git.avsdev.uk/Go/job-search/internal/mock/findATender"
"github.com/spf13/cobra"
)
// options holds the values of every command-line flag.
type options struct {
addr string
baseURL string
dataDir string
reset bool
backfill time.Duration
minGap time.Duration
maxGap time.Duration
rateLimit int
rateWindow time.Duration
accessLogPath string
logFormat string
tz string
}
const longHelp = `Mock of the Find a Tender (FTS) OCDS release package endpoint:
GET /api/1.0/ocdsReleasePackages?updatedFrom=2026-10-06T00:00:00&limit=5
Query parameters (mirroring the real service):
updatedFrom yyyy-MM-ddTHH:mm:ss releases updated at or after this time
updatedTo yyyy-MM-ddTHH:mm:ss releases updated at or before this time (optional)
limit 1..100 (default 100) page size
cursor opaque token from links.next
Like the real service, updatedFrom and updatedTo are UK local time
(Europe/London, so BST in summer); change this with --tz. Values with an
explicit offset or Z are taken as given. Release dates are stored and
returned in UTC.
Data: the catalog of buyers, suppliers, tender subjects, procurement methods
and classifications that releases are generated from is embedded in the
build from internal/mock/findATender/data/catalog.json. The mock stops with an
error if it is invalid.
Releases are generated from that catalog and appended to
<data-dir>/releases.jsonl, so they survive restarts. A background worker
publishes a new release every --min-gap to --max-gap. On startup the mock
fills any gap since the last stored release, going back at most --backfill
on a fresh data directory. Use --reset to discard stored releases.
Rate limiting: each client (by remote IP) may make --rate-limit requests per
--rate-window. Windows are fixed and anchored on the client's first request
of the current clock hour. Over the limit the mock returns 429 Too Many
Requests, a Retry-After header, and the body
"Rate limit of N exceeded. Please retry after X seconds."
Access logging: every request is written in Apache/nginx "combined" format
(or "common" with --log-format) to stdout, or to the --access-log file.
Startup and error messages go to stderr.`
func main() {
// Cancelled on Ctrl+C or SIGTERM; Cobra hands it to the command.
ctx, stop := signal.NotifyContext(
context.Background(), os.Interrupt, syscall.SIGTERM,
)
defer stop()
if err := newRootCmd().ExecuteContext(ctx); err != nil {
// Cobra has already printed the error.
stop()
os.Exit(1)
}
}
func newRootCmd() *cobra.Command {
o := &options{}
cmd := &cobra.Command{
Use: "fts-mock",
Short: "Mock of the Find a Tender OCDS release package API",
Long: longHelp,
Args: cobra.NoArgs,
SilenceUsage: true,
Example: `
fts-mock --addr :8080
fts-mock --data-dir ./mock-data --reset
fts-mock --min-gap 5s --max-gap 10s --rate-limit 1000`,
PreRunE: func(cmd *cobra.Command, args []string) error {
return o.validate()
},
RunE: func(cmd *cobra.Command, args []string) error {
return run(cmd.Context(), o)
},
}
f := cmd.Flags()
f.SortFlags = false // show flags in the order defined here
f.StringVar(&o.addr, "addr", ":8080", "listen address")
f.StringVar(
&o.baseURL,
"base-url",
"http://localhost:8080",
"public base URL used in uri and links.next",
)
f.StringVar(
&o.dataDir,
"data-dir",
"./mock-data",
"directory where releases.jsonl will be stored",
)
f.BoolVar(
&o.reset,
"reset",
false,
"delete stored releases and regenerate the --backfill history",
)
f.DurationVar(
&o.backfill,
"backfill",
72*time.Hour,
"history to generate when the data directory is empty",
)
f.DurationVar(&o.minGap, "min-gap", 30*time.Second, "minimum time between generated releases")
f.DurationVar(&o.maxGap, "max-gap", 5*time.Minute, "maximum time between generated releases")
f.IntVar(&o.rateLimit, "rate-limit", 12, "requests allowed per window per client")
f.DurationVar(&o.rateWindow, "rate-window", 5*time.Minute, "rate limit window length")
f.StringVar(&o.accessLogPath, "access-log", "-", `access log file, or "-" for stdout`)
f.StringVar(&o.logFormat, "log-format", "combined", `access log format: "combined" or "common"`)
f.StringVar(
&o.tz,
"tz",
"Europe/London",
`time zone for updatedFrom/updatedTo without an offset ("Local" for this machine's)`,
)
return cmd
}
// validate checks flag values before anything is started.
func (o *options) validate() error {
if o.logFormat != "combined" && o.logFormat != "common" {
return fmt.Errorf(`--log-format must be "combined" or "common", got %q`, o.logFormat)
}
if o.minGap <= 0 || o.maxGap < o.minGap {
return errors.New("--min-gap must be positive and no larger than --max-gap")
}
if o.rateLimit < 1 || o.rateWindow <= 0 {
return errors.New("--rate-limit must be at least 1 and --rate-window positive")
}
if _, err := time.LoadLocation(o.tz); err != nil {
return fmt.Errorf("--tz: %w", err)
}
return nil
}
// run starts the mock and blocks until ctx is cancelled or the server fails.
func run(ctx context.Context, o *options) error {
loc, err := time.LoadLocation(o.tz) // already validated
if err != nil {
return fmt.Errorf("--tz: %w", err)
}
if err := os.MkdirAll(o.dataDir, 0o750); err != nil {
return err
}
// ----- catalog -----
cat, err := mock.LoadCatalog()
if err != nil {
return err
}
log.Printf(
"catalog: %d buyers, %d suppliers, %d subjects, %d methods, %d classifications",
len(
cat.Buyers,
),
len(cat.Suppliers),
len(cat.Subjects),
len(cat.Methods),
len(cat.Classifications),
)
// ----- store -----
storePath := filepath.Join(o.dataDir, mock.ReleasesFile)
if o.reset {
if err := os.Remove(storePath); err != nil && !errors.Is(err, os.ErrNotExist) {
return err
}
log.Print("store: reset")
}
store, err := mock.OpenStore(storePath)
if err != nil {
return err
}
defer store.Close()
// ----- generator -----
gen := mock.NewGenerator(cat, store, time.Now().UnixNano())
n, err := gen.CatchUp(time.Now(), o.backfill, o.minGap, o.maxGap)
if err != nil {
return err
}
log.Printf("store: %d releases (%d generated to catch up)", store.Len(), n)
// The generator and server share a context that also ends if the server
// fails, so the generator is never left running after run returns.
ctx, cancel := context.WithCancel(ctx)
defer cancel()
workerDone := make(chan struct{})
go func() {
defer close(workerDone)
gen.Run(ctx, o.minGap, o.maxGap)
}()
// ----- HTTP -----
out := os.Stdout
if o.accessLogPath != "-" {
f, err := os.OpenFile(
o.accessLogPath, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0o600,
)
if err != nil {
cancel()
<-workerDone
return fmt.Errorf("opening access log: %w", err)
}
defer f.Close()
out = f
}
accessLogger := log.New(out, "", 0)
rl := mock.NewRateLimiter(o.rateLimit, o.rateWindow)
mux := http.NewServeMux()
mux.HandleFunc(
mock.BasePath, rl.Middleware(mock.NewHandler(store, o.baseURL, loc)),
)
httpSrv := &http.Server{
Addr: o.addr,
Handler: logging.AccessLog(accessLogger, o.logFormat == "combined", mux),
ReadHeaderTimeout: 10 * time.Second,
}
go func() {
<-ctx.Done()
shutdownCtx, cancelShutdown := context.WithTimeout(
context.Background(), 5*time.Second,
)
defer cancelShutdown()
httpSrv.Shutdown(shutdownCtx)
}()
log.Printf("FTS mock listening on %s%s (query times in %s)", o.addr, mock.BasePath, loc)
serveErr := httpSrv.ListenAndServe()
cancel() // stop the worker (and the shutdown goroutine) if the server failed
<-workerDone
if serveErr != nil && !errors.Is(serveErr, http.ErrServerClosed) {
return serveErr
}
log.Print("stopped")
return nil
}