Added fetch script for UK Gov FindATender service which is based on OCDS (with issues...)

internal/cli/fetch/command.go

@@ -0,0 +1,39 @@
+/*
+Copyright © 2026 Craig Williams <craig@avsdev.uk>
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Lesser General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Lesser General Public License for more details.
+
+You should have received a copy of the GNU Lesser General Public License
+along with this program. If not, see <http://www.gnu.org/licenses/>.
+*/
+package fetch
+
+import (
+	"log/slog"
+
+	"git.avsdev.uk/Go/job-search/internal/cli"
+	"github.com/spf13/cobra"
+)
+
+func NewCommand(opts *cli.Options) *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   "fetch",
+		Short: "Fetch current opportunities",
+		PersistentPreRun: func(cmd *cobra.Command, args []string) {
+			slog.Debug("fetch.PersistentPreRun")
+		},
+		PersistentPostRun: func(cmd *cobra.Command, args []string) {
+			slog.Debug("fetch.PersistentPostRun")
+		},
+	}
+	cmd.AddCommand(commandOnce(opts))
+	return cmd
+}

internal/cli/fetch/once.go

@@ -0,0 +1,132 @@
+/*
+Copyright © 2026 Craig Williams <craig@avsdev.uk>
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Lesser General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Lesser General Public License for more details.
+
+You should have received a copy of the GNU Lesser General Public License
+along with this program. If not, see <http://www.gnu.org/licenses/>.
+*/
+package fetch
+
+import (
+	"encoding/json"
+	"fmt"
+	"log/slog"
+	"slices"
+	"time"
+
+	"git.avsdev.uk/Go/job-search/internal/cli"
+	"git.avsdev.uk/Go/job-search/internal/findATender"
+	"git.avsdev.uk/Go/job-search/internal/findATender/ocds"
+	"github.com/spf13/cobra"
+)
+
+func filter(reqs []ocds.Release, tag string) []ocds.Release {
+	var resp []ocds.Release
+	for _, req := range reqs {
+		if slices.Contains(req.Tag, tag) {
+			resp = append(resp, req)
+		}
+	}
+	return resp
+}
+
+func commandOnce(opts *cli.Options) *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   "once",
+		Short: "Fetch current opportunities - single shot",
+		PersistentPreRun: func(cmd *cobra.Command, args []string) {
+			slog.Debug("fetch.once.PersistentPreRun")
+		},
+		PersistentPostRun: func(cmd *cobra.Command, args []string) {
+			slog.Debug("fetch.once.PersistentPostRun")
+		},
+		RunE: func(cmd *cobra.Command, args []string) error {
+			slog.Debug("fetch.once.Run")
+
+			conf, err := opts.RunSetup(cmd, true, true)
+			if err != nil {
+				return fmt.Errorf("no configuration loaded (%w)", err)
+			}
+
+			since, sinceErr := cmd.Flags().GetTime("since")
+			if sinceErr != nil {
+				return sinceErr
+			}
+
+			ft := findATender.New(conf, conf.Sources.FindATender.APIRoot)
+			defer ft.Shutdown()
+
+			ignoreRateLimit, _ := cmd.Flags().GetBool("ignore-rate-limit")
+
+			results, err := ft.FetchAll(since, !ignoreRateLimit)
+			if err != nil {
+				return err
+			}
+
+			planning, _ := cmd.Flags().GetBool("planning")
+			opportunities, _ := cmd.Flags().GetBool("opportunities")
+			awards, _ := cmd.Flags().GetBool("awards")
+
+			if (planning && (opportunities || awards)) ||
+				(opportunities && (planning || awards)) ||
+				(awards && (planning || opportunities)) {
+				return fmt.Errorf(
+					"--planning, --opportunities and --awards are exclusive, only one can be provided",
+				)
+			}
+
+			filterTag := ""
+			if planning {
+				filterTag = "planning"
+			}
+			if opportunities {
+				filterTag = "opportunity"
+			}
+			if awards {
+				filterTag = "award"
+			}
+
+			if len(filterTag) > 0 {
+				results = filter(results, filterTag)
+			}
+
+			jbytes, err := json.MarshalIndent(results, "", "  ")
+			if err != nil {
+				return err
+			}
+			fmt.Println(string(jbytes))
+
+			return nil
+		},
+	}
+	cmd.Flags().Time(
+		"since",
+		time.Time{},
+		[]string{
+			time.DateOnly, time.DateTime,
+		},
+		"contract updates published since YYYY-MM-DD[ hh:mm:ss]",
+	)
+	cmd.Flags().Bool(
+		"planning", false, "only return planning (i.e. exclude opportunities and awards)",
+	)
+	cmd.Flags().Bool(
+		"opportunities", false, "only return opportunities (i.e. exclude awards and planning)",
+	)
+	cmd.Flags().Bool(
+		"awards", false, "only return awards (i.e. exclude planning and opportunities)",
+	)
+	cmd.Flags().Bool(
+		"ignore-rate-limit", false, "ignore the rate limit and fail if exceeded",
+	)
+	return cmd
+}

internal/config/conf_source_findTender.go

@@ -0,0 +1,48 @@
+/*
+Copyright © 2026 Craig Williams <craig@avsdev.uk>
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Lesser General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Lesser General Public License for more details.
+
+You should have received a copy of the GNU Lesser General Public License
+along with this program. If not, see <http://www.gnu.org/licenses/>.
+*/
+package config
+
+import "reflect"
+
+const JSCKSourceFindATender string = "find_a_tender"
+
+type JobSearchConfigSourceFindATender struct {
+	JobSearchConfigSource
+	APIRoot string `cfg:"api_root" default:"https://www.find-tender.service.gov.uk/api/1.0"`
+}
+
+/*
+ * General section functions
+ * See related methods in section.go
+ */
+func (c JobSearchConfigSourceFindATender) Map() map[string]any { return mapOf(c) }
+
+func (c JobSearchConfigSourceFindATender) Flatten() map[string]any {
+	return flattenOf(JSCKSourceFindATender, c)
+}
+
+func (c JobSearchConfigSourceFindATender) TypeOf(key string) (reflect.Kind, error) {
+	return typeOf(c, key)
+}
+
+func (c JobSearchConfigSourceFindATender) ValueOf(key string) (any, error) {
+	return valueOf(c, key)
+}
+
+func (c *JobSearchConfigSourceFindATender) SetValue(key string, value any) error {
+	return setValue(c, key, value)
+}

internal/config/conf_sources.go

@@ -0,0 +1,83 @@
+/*
+Copyright © 2026 Craig Williams <craig@avsdev.uk>
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Lesser General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Lesser General Public License for more details.
+
+You should have received a copy of the GNU Lesser General Public License
+along with this program. If not, see <http://www.gnu.org/licenses/>.
+*/
+package config
+
+import (
+	"reflect"
+	"time"
+)
+
+const JSCKSources string = "sources"
+
+// Each field's cfg tag must match the section's JSCK* constant.
+type JobSearchConfigSources struct {
+	FindATender JobSearchConfigSourceFindATender `cfg:"find_a_tender"`
+}
+
+/*
+ * General section functions
+ * See related methods in section.go
+ */
+func (c JobSearchConfigSources) Map() map[string]any { return mapOf(c) }
+
+func (c JobSearchConfigSources) Flatten() map[string]any {
+	return flattenOf(JSCKSources, c)
+}
+
+func (c JobSearchConfigSources) TypeOf(key string) (reflect.Kind, error) {
+	return typeOf(c, key)
+}
+
+func (c JobSearchConfigSources) ValueOf(key string) (any, error) {
+	return valueOf(c, key)
+}
+
+func (c *JobSearchConfigSources) SetValue(key string, value any) error {
+	return setValue(c, key, value)
+}
+
+/*
+ *** Generic source definition, embedded in each source ***
+ */
+type JobSearchConfigSource struct {
+	Enabled  bool          `cfg:"enabled"`
+	Interval time.Duration `cfg:"interval" default:"10m"`
+}
+
+/*
+ * General section functions
+ * See related methods in section.go
+ */
+func (c *JobSearchConfigSource) Keys() []string { return keysOf(c) }
+
+func (c JobSearchConfigSource) Map() map[string]any { return mapOf(c) }
+
+func (c JobSearchConfigSource) Flatten() map[string]any {
+	return flattenOf("", c)
+}
+
+func (c JobSearchConfigSource) TypeOf(key string) (reflect.Kind, error) {
+	return typeOf(c, key)
+}
+
+func (c JobSearchConfigSource) ValueOf(key string) (any, error) {
+	return valueOf(c, key)
+}
+
+func (c *JobSearchConfigSource) SetValue(key string, value any) error {
+	return setValue(c, key, value)
+}

internal/config/config.go

@@ -32,7 +32,8 @@ const EtcConfigPath string = "/etc/job-search"
 
 // Each field's cfg tag must match the section's JSCK* constant.
 type JobSearchConfig struct {
-	Logging  JobSearchConfigLogging  `cfg:"logging"`
+	Logging JobSearchConfigLogging `cfg:"logging"`
+	Sources JobSearchConfigSources `cfg:"sources"`
 }
 
 // Default configuration, taken from the default struct tags

internal/findATender/.gitignore

@@ -0,0 +1 @@
+ocds_gen.go

internal/findATender/api.go

@@ -0,0 +1,218 @@
+/*
+Copyright © 2026 Craig Williams <craig@avsdev.uk>
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Lesser General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Lesser General Public License for more details.
+
+You should have received a copy of the GNU Lesser General Public License
+along with this program. If not, see <http://www.gnu.org/licenses/>.
+*/
+package findATender
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"log/slog"
+	"strconv"
+	"time"
+
+	"git.avsdev.uk/Go/job-search/internal/findATender/ocds"
+	"git.avsdev.uk/Go/job-search/internal/logging"
+	resty "resty.dev/v3"
+)
+
+var ErrNoNext = errors.New("no next link")
+var ErrNoReleases = errors.New("no releases")
+var ErrRateLimitExceeded = errors.New("rate limit has been exceeded")
+var ErrFetchCancelled = errors.New("fetch cancelled")
+
+type RequestOptions struct {
+	Since          time.Time
+	Limit          int
+	AllowRateLimit bool
+}
+
+func rateSleep(ctx context.Context, duration time.Duration) error {
+	timer := time.NewTimer(duration)
+	defer timer.Stop()
+
+	select {
+	case <-timer.C:
+		return nil
+	case <-ctx.Done():
+		return ctx.Err()
+	}
+}
+
+func fetchURI(ft *FindATender, uri string) (*resty.Response, error) {
+	withTrace := ft.config.Logging.LogLevel() <= slog.LevelDebug
+
+	slog.Info("findTender.resty", "request_uri", uri)
+
+	ctx := ft.ctx
+	if ctx == nil {
+		ctx = context.Background()
+	}
+
+	req := ft.httpClient.R()
+	req.SetContext(ctx)
+	res, err := req.SetTrace(withTrace).Get(uri)
+
+	if withTrace {
+		if logging.IsJSONHandler(slog.Default().Handler()) {
+			slog.Debug(
+				"Resty",
+				"trace", res.Request.TraceInfo().JSON(),
+				"status", res.Status(),
+				"response", res,
+			)
+		} else {
+			slog.Debug("Resty " + res.Request.TraceInfo().String())
+			slog.Debug("Resty status: " + res.Status())
+			slog.Debug("Resty response: " + res.String())
+		}
+	}
+
+	return res, err
+}
+
+func (ft *FindATender) fetchURI(
+	uri string,
+	allowRateLimit bool,
+) (*resty.Response, error) {
+	res, err := fetchURI(ft, uri)
+	if err != nil {
+		return nil, err
+	}
+
+	if ft.ctx == nil {
+		copied := *ft
+		copied.ctx = context.Background()
+		ft = &copied
+	}
+
+	if res.StatusCode() == 429 {
+		if !allowRateLimit {
+			return nil, ErrRateLimitExceeded
+		}
+
+		retrySecStr := res.Header().Get("Retry-After")
+		retrySec, err := strconv.ParseInt(retrySecStr, 10, 32)
+		if err != nil {
+			return nil, fmt.Errorf("unknown retry-after value: %w", err)
+		}
+
+		slog.Warn(fmt.Sprintf(
+			"API rate limit hit, waiting as requested for %d seconds", retrySec,
+		))
+		if rateSleep(ft.ctx, time.Duration(retrySec)*time.Second) != nil {
+			return nil, ErrFetchCancelled
+		}
+
+		res, err = fetchURI(ft, uri)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	return res, nil
+}
+
+func (ft *FindATender) fetchWithOpts(
+	opts RequestOptions,
+) (*resty.Response, error) {
+	if opts.Since.IsZero() {
+		opts.Since = time.Now().AddDate(0, 0, 1)
+	}
+	if opts.Limit == 0 {
+		opts.Limit = 100
+	}
+
+	uri := fmt.Sprintf(
+		"%s/ocdsReleasePackages?updatedFrom=%s&limit=%d",
+		ft.apiRoot,
+		opts.Since.Format("2006-01-02T15:04:05"),
+		opts.Limit,
+	)
+
+	return ft.fetchURI(uri, opts.AllowRateLimit)
+}
+
+func (ft *FindATender) fetchNext(
+	pkg ocds.ReleasePackage,
+	allowRateLimit bool,
+) (*resty.Response, error) {
+	if pkg.Links == nil || len(pkg.Links.Next) == 0 {
+		return nil, ErrNoNext
+	}
+	return ft.fetchURI(pkg.Links.Next, allowRateLimit)
+}
+
+func (ft *FindATender) fetchOnce(opts RequestOptions) ([]ocds.Release, error) {
+	res, err := ft.fetchWithOpts(opts)
+	if err != nil {
+		return nil, err
+	}
+	defer res.Body.Close()
+
+	var relPkg ocds.ReleasePackage
+	decoder := json.NewDecoder(NewOCDSStreamCleaner(res.Body))
+	if err := decoder.Decode(&relPkg); err != nil {
+		return nil, err
+	}
+
+	if len(relPkg.Releases) == 0 {
+		return nil, ErrNoReleases
+	}
+
+	return relPkg.Releases, nil
+}
+
+func (ft *FindATender) fetchAll(opts RequestOptions) ([]ocds.Release, error) {
+	var releases []ocds.Release
+
+	res, err := ft.fetchWithOpts(opts)
+	if err != nil {
+		return nil, err
+	}
+	defer res.Body.Close()
+
+	var relPkg ocds.ReleasePackage
+	decoder := json.NewDecoder(NewOCDSStreamCleaner(res.Body))
+	if err := decoder.Decode(&relPkg); err != nil {
+		return nil, err
+	}
+	releases = append(releases, relPkg.Releases...)
+
+	for true {
+		res, err := ft.fetchNext(relPkg, opts.AllowRateLimit)
+		if err == ErrNoNext {
+			break
+		}
+		if err != nil {
+			return nil, err
+		}
+		defer res.Body.Close()
+
+		decoder := json.NewDecoder(NewOCDSStreamCleaner(res.Body))
+		if err := decoder.Decode(&relPkg); err != nil {
+			return nil, err
+		}
+		releases = append(releases, relPkg.Releases...)
+	}
+
+	if len(releases) == 0 {
+		return nil, ErrNoReleases
+	}
+
+	return releases, nil
+}

internal/findATender/dateRepair.go

@@ -0,0 +1,59 @@
+/*
+Copyright © 2026 Craig Williams <craig@avsdev.uk>
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Lesser General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Lesser General Public License for more details.
+
+You should have received a copy of the GNU Lesser General Public License
+along with this program. If not, see <http://www.gnu.org/licenses/>.
+*/
+package findATender
+
+import (
+	"bufio"
+	"bytes"
+	"io"
+	"regexp"
+)
+
+// Global regex to identify and catch date-only fields
+var invalidDateRegex = regexp.MustCompile(`:\s*"(\d{4}-\d{2}-\d{2})"\s*([,}\s])`)
+var invalidPublishedDateRegex = regexp.MustCompile(`"publishedDate":\s*""`)
+
+// The UK Government FindATender service doesn't strictly follow OCDS date
+// format. Sometimes it provides dates only, sometimes it sends empty strings
+// instead of nulls.
+type FindATenderStreamCleaner struct {
+	reader *bufio.Reader
+	buffer bytes.Buffer
+}
+
+func NewOCDSStreamCleaner(r io.Reader) *FindATenderStreamCleaner {
+	return &FindATenderStreamCleaner{reader: bufio.NewReader(r)}
+}
+
+func (sc *FindATenderStreamCleaner) Read(p []byte) (int, error) {
+	if sc.buffer.Len() == 0 {
+		line, err := sc.reader.ReadBytes('\n')
+		if len(line) > 0 {
+			// Convert "2026-11-16" -> "2026-11-16T00:00:00Z"
+			line = invalidDateRegex.ReplaceAll(line, []byte(`:"$1T00:00:00Z"`))
+			// Convert {publishedDate:""} -> "{publishedDate:null}"
+			line = invalidPublishedDateRegex.ReplaceAll(line, []byte(`"publishedDate":null`))
+			sc.buffer.Write(line)
+		}
+		if err != nil {
+			if sc.buffer.Len() == 0 {
+				return 0, err
+			}
+		}
+	}
+	return sc.buffer.Read(p)
+}

internal/findATender/findATender.go

@@ -0,0 +1,91 @@
+/*
+Copyright © 2026 Craig Williams <craig@avsdev.uk>
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Lesser General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Lesser General Public License for more details.
+
+You should have received a copy of the GNU Lesser General Public License
+along with this program. If not, see <http://www.gnu.org/licenses/>.
+*/
+package findATender
+
+import (
+	"context"
+	"time"
+
+	"git.avsdev.uk/Go/job-search/internal/config"
+	"git.avsdev.uk/Go/job-search/internal/findATender/ocds"
+	resty "resty.dev/v3"
+)
+
+type FindATender struct {
+	httpClient *resty.Client
+	config     *config.JobSearchConfig
+	apiRoot    string
+	ctx        context.Context
+}
+
+func New(conf *config.JobSearchConfig, apiRoot string) FindATender {
+	ft := FindATender{
+		config:  conf,
+		apiRoot: apiRoot,
+	}
+	ft.httpClient = resty.New().
+		SetTimeout(60 * time.Duration(time.Second)).
+		SetRetryCount(1)
+	return ft
+}
+
+func (ft *FindATender) ReloadConfig(conf *config.JobSearchConfig) {
+	ft.config = conf
+	// TODO: any reloading required for configuration change handling
+}
+
+func (ft *FindATender) With(ctx context.Context) *FindATender {
+	if ctx == nil {
+		return ft
+	}
+
+	copied := *ft
+	copied.ctx = ctx
+	return &copied
+}
+
+func (ft *FindATender) Shutdown() {
+	ft.httpClient.Close()
+}
+
+func (ft *FindATender) FetchOnce(
+	since time.Time,
+	allowRateLimit bool,
+) ([]ocds.Release, error) {
+	reqs, err := ft.fetchOnce(RequestOptions{
+		Since:          since,
+		AllowRateLimit: allowRateLimit,
+	})
+	if err != nil {
+		return nil, err
+	}
+	return reqs, nil
+}
+
+func (ft *FindATender) FetchAll(
+	since time.Time,
+	allowRateLimit bool,
+) ([]ocds.Release, error) {
+	reqs, err := ft.fetchAll(RequestOptions{
+		Since:          since,
+		AllowRateLimit: allowRateLimit,
+	})
+	if err != nil {
+		return nil, err
+	}
+	return reqs, nil
+}

internal/findATender/ocds/extensions.json

@@ -0,0 +1,16 @@
+{
+  "releaseSchema": "https://raw.githubusercontent.com/open-contracting/standard/1.1/schema/release-schema.json",
+  "releasePackageSchema": "https://raw.githubusercontent.com/open-contracting/standard/1.1/schema/release-package-schema.json",
+  "extensions": [
+    "https://standard.open-contracting.org/profiles/eu/latest/en/extension.json",
+    "https://raw.githubusercontent.com/open-contracting-extensions/ocds_amendment_rationale_classifications_extension/master/extension.json",
+    "https://raw.githubusercontent.com/open-contracting-extensions/ocds_budget_breakdown_extension/master/extension.json",
+    "https://raw.githubusercontent.com/open-contracting-extensions/ocds_contract_completion_extension/master/extension.json",
+    "https://raw.githubusercontent.com/open-contracting-extensions/ocds_documentation_extension/master/extension.json",
+    "https://raw.githubusercontent.com/open-contracting-extensions/ocds_pagination_extension/master/extension.json",
+    "https://raw.githubusercontent.com/open-contracting-extensions/ocds_suitability_extension/master/extension.json",
+    "https://raw.githubusercontent.com/portaledcahn/ocds_Links_extension/main/extension.json",
+    "https://raw.githubusercontent.com/cabinetoffice/ocds_uk_extension/main/extension.json",
+    "https://raw.githubusercontent.com/open-contracting-extensions/ocds_performance_failures_extension/master/extension.json"
+  ]
+}

internal/findATender/ocds/generate.go

@@ -0,0 +1,5 @@
+// Package full contains OCDS 1.1 types with the full extension set applied:
+// the EU profile, UK extension, and the other extensions in extensions.json.
+package ocds
+
+//go:generate go run git.avsdev.uk/Go/go-ocds/cmd/ocdsgen generate -c extensions.json -o ocds_gen.go -p ocds