Go/job-search
1
0 Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases 2 Wiki Activity

Added fetch script for UK Gov FindATender service which is based on OCDS (with issues...)

Browse Source
This commit is contained in:
bwilliams committed 2026-10-09 10:08:11 +01:00
1 parent ff33ef082a
commit e43b7a8370
16 files changed
+743 -10
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

README.md
+14
View File
@@ -2,3 +2,17 @@
JobSearch is a service for scraping for contract opportunities. It is primarily
a re-write of the existing nodeJS JobSearch codebase.
## Installation of dev tools
A number of tools have been used in the development of the project. Installation
of these tools can be done using `go install`.
To install the tools:
```bash
go install github.com/spf13/cobra-cli@latest
go install git.avsdev.uk/Go/go-ocds/cmd/ocdsgen@latest
```
Ensure that the GOBIN path ($HOME/go/bin) is added to the environment before
running any of the tools.
cmd/jobsearch/main.go
+2
View File
@@ -26,6 +26,7 @@ import (
"git.avsdev.uk/Go/job-search/internal/cli"
"git.avsdev.uk/Go/job-search/internal/cli/config"
"git.avsdev.uk/Go/job-search/internal/cli/fetch"
"git.avsdev.uk/Go/job-search/internal/cli/version"
"github.com/spf13/cobra"
)
@@ -72,6 +73,7 @@ opportunities.
root.AddCommand(
version.NewCommand(),
config.NewCommand(opts),
fetch.NewCommand(opts),
)
return root
go.mod
+7 -3
View File
@@ -3,15 +3,17 @@ module git.avsdev.uk/Go/job-search
go 1.26.8
require (
git.avsdev.uk/Go/go-ocds v1.0.0
github.com/joho/godotenv v1.5.1
github.com/lmittmann/tint v1.2.0
github.com/spf13/cobra v1.10.2
github.com/spf13/viper v1.21.0
go.yaml.in/yaml/v3 v3.0.5
resty.dev/v3 v3.0.0-rc.4
)
require (
github.com/fsnotify/fsnotify v1.9.0 // indirect
github.com/fsnotify/fsnotify v1.10.1 // indirect
github.com/go-viper/mapstructure/v2 v2.4.0 // indirect
github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/pelletier/go-toml/v2 v2.2.4 // indirect
@@ -20,7 +22,9 @@ require (
github.com/spf13/afero v1.15.0 // indirect
github.com/spf13/cast v1.10.0 // indirect
github.com/spf13/pflag v1.0.10 // indirect
github.com/stretchr/testify v1.12.1 // indirect
github.com/subosito/gotenv v1.6.0 // indirect
golang.org/x/sys v0.29.0 // indirect
golang.org/x/text v0.28.0 // indirect
golang.org/x/net v0.58.0 // indirect
golang.org/x/sys v0.47.0 // indirect
golang.org/x/text v0.41.0 // indirect
)
go.sum
+24 -7
View File
@@ -1,16 +1,28 @@
git.avsdev.uk/Go/go-ocds v1.0.0 h1:X/CoCyDcVy8ObGZ+UK4t6viPGj2bqATTEYc4Q6/2DKA=
git.avsdev.uk/Go/go-ocds v1.0.0/go.mod h1:YPnqdf7ft6vPHiHLzeL/R7RtWYNKZJF1NXu/YEBpEbA=
github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k=
github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0=
github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8=
github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
github.com/fsnotify/fsnotify v1.10.1 h1:b0/UzAf9yR5rhf3RPm9gf3ehBPpf0oZKIjtpKrx59Ho=
github.com/fsnotify/fsnotify v1.10.1/go.mod h1:TLheqan6HD6GBK6PrDWyDPBaEV8LspOxvPSjC+bVfgo=
github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9LvH92wZUgs=
github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM=
github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0=
github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/lmittmann/tint v1.2.0 h1:AogHRHy8HUJUnNJBHJlYa+fR4YY8mko2cnCp67xn9JY=
github.com/lmittmann/tint v1.2.0/go.mod h1:HIS3gSy7qNwGCj+5oRjAutErFBl4BzdQP6cJZ0NfMwE=
github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4=
github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY=
github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8=
github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/sagikazarmark/locafero v0.11.0 h1:1iurJgmM9G3PA/I+wWYIOw/5SyBtxapeHDcg+AAIFXc=
github.com/sagikazarmark/locafero v0.11.0/go.mod h1:nVIGvgyzw595SUSUE6tvCp3YYTeHs15MvlmU87WwIik=
@@ -22,19 +34,24 @@ github.com/spf13/cast v1.10.0 h1:h2x0u2shc1QuLHfxi+cTJvs30+ZAHOGRic8uyGTDWxY=
github.com/spf13/cast v1.10.0/go.mod h1:jNfB8QC9IA6ZuY2ZjDp0KtFO2LZZlg4S/7bzP6qqeHo=
github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU=
github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4=
github.com/spf13/pflag v1.0.9 h1:9exaQaMOCwffKiiiYk6/BndUBv+iRViNW+4lEMi0PvY=
github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk=
github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
github.com/spf13/viper v1.21.0 h1:x5S+0EU27Lbphp4UKm1C+1oQO+rKx36vfCoaVebLFSU=
github.com/spf13/viper v1.21.0/go.mod h1:P0lhsswPGWD/1lZJ9ny3fYnVqxiegrlNrEmgLjbTCAY=
github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE=
github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg=
github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8=
github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU=
go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw=
go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg=
golang.org/x/sys v0.29.0 h1:TPYlXGxvx1MGTn2GiZDhnjPA9wZzZeGKHHmKhHYvgaU=
golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/text v0.28.0 h1:rhazDwis8INMIwQ4tpjLDzUhx6RlXqZNPEM0huQojng=
golang.org/x/text v0.28.0/go.mod h1:U8nCwOR8jO/marOQ0QbDiOngZVEBB7MAiitBuMjXiNU=
golang.org/x/net v0.58.0 h1:ynWG7rqYi4ccpTEuPZ2QGWHktVEM9DMCj9yzDE0Q7To=
golang.org/x/net v0.58.0/go.mod h1:YwCddHnFlT7eLQqVprV19OnhLGtc5xOKgE0RyqgfWAU=
golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs=
golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
golang.org/x/text v0.41.0 h1:vz/seA0lnX87Othu2f/0L24RcgrXD9/YFTSuGjj3rH8=
golang.org/x/text v0.41.0/go.mod h1:jvf1O8ajNzZqhSrQBPbutR/EB83Cc0CFrezNQIwbb5M=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
resty.dev/v3 v3.0.0-rc.4 h1:n3360yEcFiyrX5Vkc9XG3Y8kMM24alRBxJ9Ev1mLmqs=
resty.dev/v3 v3.0.0-rc.4/go.mod h1:NTOerrC/4T7/FE6tXIZGIysXXBdgNqwMZuKtxpea9NM=
internal/cli/fetch/command.go
+39
View File
@@ -0,0 +1,39 @@
/*
Copyright © 2026 Craig Williams <craig@avsdev.uk>
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package fetch
import (
"log/slog"
"git.avsdev.uk/Go/job-search/internal/cli"
"github.com/spf13/cobra"
)
func NewCommand(opts *cli.Options) *cobra.Command {
cmd := &cobra.Command{
Use: "fetch",
Short: "Fetch current opportunities",
PersistentPreRun: func(cmd *cobra.Command, args []string) {
slog.Debug("fetch.PersistentPreRun")
},
PersistentPostRun: func(cmd *cobra.Command, args []string) {
slog.Debug("fetch.PersistentPostRun")
},
}
cmd.AddCommand(commandOnce(opts))
return cmd
}
internal/cli/fetch/once.go
+132
View File
@@ -0,0 +1,132 @@
/*
Copyright © 2026 Craig Williams <craig@avsdev.uk>
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package fetch
import (
"encoding/json"
"fmt"
"log/slog"
"slices"
"time"
"git.avsdev.uk/Go/job-search/internal/cli"
"git.avsdev.uk/Go/job-search/internal/findATender"
"git.avsdev.uk/Go/job-search/internal/findATender/ocds"
"github.com/spf13/cobra"
)
func filter(reqs []ocds.Release, tag string) []ocds.Release {
var resp []ocds.Release
for _, req := range reqs {
if slices.Contains(req.Tag, tag) {
resp = append(resp, req)
}
}
return resp
}
func commandOnce(opts *cli.Options) *cobra.Command {
cmd := &cobra.Command{
Use: "once",
Short: "Fetch current opportunities - single shot",
PersistentPreRun: func(cmd *cobra.Command, args []string) {
slog.Debug("fetch.once.PersistentPreRun")
},
PersistentPostRun: func(cmd *cobra.Command, args []string) {
slog.Debug("fetch.once.PersistentPostRun")
},
RunE: func(cmd *cobra.Command, args []string) error {
slog.Debug("fetch.once.Run")
conf, err := opts.RunSetup(cmd, true, true)
if err != nil {
return fmt.Errorf("no configuration loaded (%w)", err)
}
since, sinceErr := cmd.Flags().GetTime("since")
if sinceErr != nil {
return sinceErr
}
ft := findATender.New(conf, conf.Sources.FindATender.APIRoot)
defer ft.Shutdown()
ignoreRateLimit, _ := cmd.Flags().GetBool("ignore-rate-limit")
results, err := ft.FetchAll(since, !ignoreRateLimit)
if err != nil {
return err
}
planning, _ := cmd.Flags().GetBool("planning")
opportunities, _ := cmd.Flags().GetBool("opportunities")
awards, _ := cmd.Flags().GetBool("awards")
if (planning && (opportunities || awards)) ||
(opportunities && (planning || awards)) ||
(awards && (planning || opportunities)) {
return fmt.Errorf(
"--planning, --opportunities and --awards are exclusive, only one can be provided",
)
}
filterTag := ""
if planning {
filterTag = "planning"
}
if opportunities {
filterTag = "opportunity"
}
if awards {
filterTag = "award"
}
if len(filterTag) > 0 {
results = filter(results, filterTag)
}
jbytes, err := json.MarshalIndent(results, "", " ")
if err != nil {
return err
}
fmt.Println(string(jbytes))
return nil
},
}
cmd.Flags().Time(
"since",
time.Time{},
[]string{
time.DateOnly, time.DateTime,
},
"contract updates published since YYYY-MM-DD[ hh:mm:ss]",
)
cmd.Flags().Bool(
"planning", false, "only return planning (i.e. exclude opportunities and awards)",
)
cmd.Flags().Bool(
"opportunities", false, "only return opportunities (i.e. exclude awards and planning)",
)
cmd.Flags().Bool(
"awards", false, "only return awards (i.e. exclude planning and opportunities)",
)
cmd.Flags().Bool(
"ignore-rate-limit", false, "ignore the rate limit and fail if exceeded",
)
return cmd
}
internal/config/conf_source_findTender.go
+48
View File
@@ -0,0 +1,48 @@
/*
Copyright © 2026 Craig Williams <craig@avsdev.uk>
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package config
import "reflect"
const JSCKSourceFindATender string = "find_a_tender"
type JobSearchConfigSourceFindATender struct {
JobSearchConfigSource
APIRoot string `cfg:"api_root" default:"https://www.find-tender.service.gov.uk/api/1.0"`
}
/*
* General section functions
* See related methods in section.go
*/
func (c JobSearchConfigSourceFindATender) Map() map[string]any { return mapOf(c) }
func (c JobSearchConfigSourceFindATender) Flatten() map[string]any {
return flattenOf(JSCKSourceFindATender, c)
}
func (c JobSearchConfigSourceFindATender) TypeOf(key string) (reflect.Kind, error) {
return typeOf(c, key)
}
func (c JobSearchConfigSourceFindATender) ValueOf(key string) (any, error) {
return valueOf(c, key)
}
func (c *JobSearchConfigSourceFindATender) SetValue(key string, value any) error {
return setValue(c, key, value)
}
internal/config/conf_sources.go
+83
View File
@@ -0,0 +1,83 @@
/*
Copyright © 2026 Craig Williams <craig@avsdev.uk>
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package config
import (
"reflect"
"time"
)
const JSCKSources string = "sources"
// Each field's cfg tag must match the section's JSCK* constant.
type JobSearchConfigSources struct {
FindATender JobSearchConfigSourceFindATender `cfg:"find_a_tender"`
}
/*
* General section functions
* See related methods in section.go
*/
func (c JobSearchConfigSources) Map() map[string]any { return mapOf(c) }
func (c JobSearchConfigSources) Flatten() map[string]any {
return flattenOf(JSCKSources, c)
}
func (c JobSearchConfigSources) TypeOf(key string) (reflect.Kind, error) {
return typeOf(c, key)
}
func (c JobSearchConfigSources) ValueOf(key string) (any, error) {
return valueOf(c, key)
}
func (c *JobSearchConfigSources) SetValue(key string, value any) error {
return setValue(c, key, value)
}
/*
*** Generic source definition, embedded in each source ***
*/
type JobSearchConfigSource struct {
Enabled bool `cfg:"enabled"`
Interval time.Duration `cfg:"interval" default:"10m"`
}
/*
* General section functions
* See related methods in section.go
*/
func (c *JobSearchConfigSource) Keys() []string { return keysOf(c) }
func (c JobSearchConfigSource) Map() map[string]any { return mapOf(c) }
func (c JobSearchConfigSource) Flatten() map[string]any {
return flattenOf("", c)
}
func (c JobSearchConfigSource) TypeOf(key string) (reflect.Kind, error) {
return typeOf(c, key)
}
func (c JobSearchConfigSource) ValueOf(key string) (any, error) {
return valueOf(c, key)
}
func (c *JobSearchConfigSource) SetValue(key string, value any) error {
return setValue(c, key, value)
}
internal/config/config.go
+1
View File
@@ -33,6 +33,7 @@ const EtcConfigPath string = "/etc/job-search"
// Each field's cfg tag must match the section's JSCK* constant.
type JobSearchConfig struct {
Logging JobSearchConfigLogging `cfg:"logging"`
Sources JobSearchConfigSources `cfg:"sources"`
}
// Default configuration, taken from the default struct tags
internal/findATender/.gitignore
+1
View File
@@ -0,0 +1 @@
ocds_gen.go
internal/findATender/api.go
+218
View File
@@ -0,0 +1,218 @@
/*
Copyright © 2026 Craig Williams <craig@avsdev.uk>
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package findATender
import (
"context"
"encoding/json"
"errors"
"fmt"
"log/slog"
"strconv"
"time"
"git.avsdev.uk/Go/job-search/internal/findATender/ocds"
"git.avsdev.uk/Go/job-search/internal/logging"
resty "resty.dev/v3"
)
var ErrNoNext = errors.New("no next link")
var ErrNoReleases = errors.New("no releases")
var ErrRateLimitExceeded = errors.New("rate limit has been exceeded")
var ErrFetchCancelled = errors.New("fetch cancelled")
type RequestOptions struct {
Since time.Time
Limit int
AllowRateLimit bool
}
func rateSleep(ctx context.Context, duration time.Duration) error {
timer := time.NewTimer(duration)
defer timer.Stop()
select {
case <-timer.C:
return nil
case <-ctx.Done():
return ctx.Err()
}
}
func fetchURI(ft *FindATender, uri string) (*resty.Response, error) {
withTrace := ft.config.Logging.LogLevel() <= slog.LevelDebug
slog.Info("findTender.resty", "request_uri", uri)
ctx := ft.ctx
if ctx == nil {
ctx = context.Background()
}
req := ft.httpClient.R()
req.SetContext(ctx)
res, err := req.SetTrace(withTrace).Get(uri)
if withTrace {
if logging.IsJSONHandler(slog.Default().Handler()) {
slog.Debug(
"Resty",
"trace", res.Request.TraceInfo().JSON(),
"status", res.Status(),
"response", res,
)
} else {
slog.Debug("Resty " + res.Request.TraceInfo().String())
slog.Debug("Resty status: " + res.Status())
slog.Debug("Resty response: " + res.String())
}
}
return res, err
}
func (ft *FindATender) fetchURI(
uri string,
allowRateLimit bool,
) (*resty.Response, error) {
res, err := fetchURI(ft, uri)
if err != nil {
return nil, err
}
if ft.ctx == nil {
copied := *ft
copied.ctx = context.Background()
ft = &copied
}
if res.StatusCode() == 429 {
if !allowRateLimit {
return nil, ErrRateLimitExceeded
}
retrySecStr := res.Header().Get("Retry-After")
retrySec, err := strconv.ParseInt(retrySecStr, 10, 32)
if err != nil {
return nil, fmt.Errorf("unknown retry-after value: %w", err)
}
slog.Warn(fmt.Sprintf(
"API rate limit hit, waiting as requested for %d seconds", retrySec,
))
if rateSleep(ft.ctx, time.Duration(retrySec)*time.Second) != nil {
return nil, ErrFetchCancelled
}
res, err = fetchURI(ft, uri)
if err != nil {
return nil, err
}
}
return res, nil
}
func (ft *FindATender) fetchWithOpts(
opts RequestOptions,
) (*resty.Response, error) {
if opts.Since.IsZero() {
opts.Since = time.Now().AddDate(0, 0, 1)
}
if opts.Limit == 0 {
opts.Limit = 100
}
uri := fmt.Sprintf(
"%s/ocdsReleasePackages?updatedFrom=%s&limit=%d",
ft.apiRoot,
opts.Since.Format("2006-01-02T15:04:05"),
opts.Limit,
)
return ft.fetchURI(uri, opts.AllowRateLimit)
}
func (ft *FindATender) fetchNext(
pkg ocds.ReleasePackage,
allowRateLimit bool,
) (*resty.Response, error) {
if pkg.Links == nil || len(pkg.Links.Next) == 0 {
return nil, ErrNoNext
}
return ft.fetchURI(pkg.Links.Next, allowRateLimit)
}
func (ft *FindATender) fetchOnce(opts RequestOptions) ([]ocds.Release, error) {
res, err := ft.fetchWithOpts(opts)
if err != nil {
return nil, err
}
defer res.Body.Close()
var relPkg ocds.ReleasePackage
decoder := json.NewDecoder(NewOCDSStreamCleaner(res.Body))
if err := decoder.Decode(&relPkg); err != nil {
return nil, err
}
if len(relPkg.Releases) == 0 {
return nil, ErrNoReleases
}
return relPkg.Releases, nil
}
func (ft *FindATender) fetchAll(opts RequestOptions) ([]ocds.Release, error) {
var releases []ocds.Release
res, err := ft.fetchWithOpts(opts)
if err != nil {
return nil, err
}
defer res.Body.Close()
var relPkg ocds.ReleasePackage
decoder := json.NewDecoder(NewOCDSStreamCleaner(res.Body))
if err := decoder.Decode(&relPkg); err != nil {
return nil, err
}
releases = append(releases, relPkg.Releases...)
for true {
res, err := ft.fetchNext(relPkg, opts.AllowRateLimit)
if err == ErrNoNext {
break
}
if err != nil {
return nil, err
}
defer res.Body.Close()
decoder := json.NewDecoder(NewOCDSStreamCleaner(res.Body))
if err := decoder.Decode(&relPkg); err != nil {
return nil, err
}
releases = append(releases, relPkg.Releases...)
}
if len(releases) == 0 {
return nil, ErrNoReleases
}
return releases, nil
}
internal/findATender/dateRepair.go
+59
View File
@@ -0,0 +1,59 @@
/*
Copyright © 2026 Craig Williams <craig@avsdev.uk>
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package findATender
import (
"bufio"
"bytes"
"io"
"regexp"
)
// Global regex to identify and catch date-only fields
var invalidDateRegex = regexp.MustCompile(`:\s*"(\d{4}-\d{2}-\d{2})"\s*([,}\s])`)
var invalidPublishedDateRegex = regexp.MustCompile(`"publishedDate":\s*""`)
// The UK Government FindATender service doesn't strictly follow OCDS date
// format. Sometimes it provides dates only, sometimes it sends empty strings
// instead of nulls.
type FindATenderStreamCleaner struct {
reader *bufio.Reader
buffer bytes.Buffer
}
func NewOCDSStreamCleaner(r io.Reader) *FindATenderStreamCleaner {
return &FindATenderStreamCleaner{reader: bufio.NewReader(r)}
}
func (sc *FindATenderStreamCleaner) Read(p []byte) (int, error) {
if sc.buffer.Len() == 0 {
line, err := sc.reader.ReadBytes('\n')
if len(line) > 0 {
// Convert "2026-11-16" -> "2026-11-16T00:00:00Z"
line = invalidDateRegex.ReplaceAll(line, []byte(`:"$1T00:00:00Z"`))
// Convert {publishedDate:""} -> "{publishedDate:null}"
line = invalidPublishedDateRegex.ReplaceAll(line, []byte(`"publishedDate":null`))
sc.buffer.Write(line)
}
if err != nil {
if sc.buffer.Len() == 0 {
return 0, err
}
}
}
return sc.buffer.Read(p)
}
internal/findATender/findATender.go
+91
View File
@@ -0,0 +1,91 @@
/*
Copyright © 2026 Craig Williams <craig@avsdev.uk>
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package findATender
import (
"context"
"time"
"git.avsdev.uk/Go/job-search/internal/config"
"git.avsdev.uk/Go/job-search/internal/findATender/ocds"
resty "resty.dev/v3"
)
type FindATender struct {
httpClient *resty.Client
config *config.JobSearchConfig
apiRoot string
ctx context.Context
}
func New(conf *config.JobSearchConfig, apiRoot string) FindATender {
ft := FindATender{
config: conf,
apiRoot: apiRoot,
}
ft.httpClient = resty.New().
SetTimeout(60 * time.Duration(time.Second)).
SetRetryCount(1)
return ft
}
func (ft *FindATender) ReloadConfig(conf *config.JobSearchConfig) {
ft.config = conf
// TODO: any reloading required for configuration change handling
}
func (ft *FindATender) With(ctx context.Context) *FindATender {
if ctx == nil {
return ft
}
copied := *ft
copied.ctx = ctx
return &copied
}
func (ft *FindATender) Shutdown() {
ft.httpClient.Close()
}
func (ft *FindATender) FetchOnce(
since time.Time,
allowRateLimit bool,
) ([]ocds.Release, error) {
reqs, err := ft.fetchOnce(RequestOptions{
Since: since,
AllowRateLimit: allowRateLimit,
})
if err != nil {
return nil, err
}
return reqs, nil
}
func (ft *FindATender) FetchAll(
since time.Time,
allowRateLimit bool,
) ([]ocds.Release, error) {
reqs, err := ft.fetchAll(RequestOptions{
Since: since,
AllowRateLimit: allowRateLimit,
})
if err != nil {
return nil, err
}
return reqs, nil
}
internal/findATender/ocds/extensions.json
+16
View File
@@ -0,0 +1,16 @@
{
"releaseSchema": "https://raw.githubusercontent.com/open-contracting/standard/1.1/schema/release-schema.json",
"releasePackageSchema": "https://raw.githubusercontent.com/open-contracting/standard/1.1/schema/release-package-schema.json",
"extensions": [
"https://standard.open-contracting.org/profiles/eu/latest/en/extension.json",
"https://raw.githubusercontent.com/open-contracting-extensions/ocds_amendment_rationale_classifications_extension/master/extension.json",
"https://raw.githubusercontent.com/open-contracting-extensions/ocds_budget_breakdown_extension/master/extension.json",
"https://raw.githubusercontent.com/open-contracting-extensions/ocds_contract_completion_extension/master/extension.json",
"https://raw.githubusercontent.com/open-contracting-extensions/ocds_documentation_extension/master/extension.json",
"https://raw.githubusercontent.com/open-contracting-extensions/ocds_pagination_extension/master/extension.json",
"https://raw.githubusercontent.com/open-contracting-extensions/ocds_suitability_extension/master/extension.json",
"https://raw.githubusercontent.com/portaledcahn/ocds_Links_extension/main/extension.json",
"https://raw.githubusercontent.com/cabinetoffice/ocds_uk_extension/main/extension.json",
"https://raw.githubusercontent.com/open-contracting-extensions/ocds_performance_failures_extension/master/extension.json"
]
}
internal/findATender/ocds/generate.go
+5
View File
@@ -0,0 +1,5 @@
// Package full contains OCDS 1.1 types with the full extension set applied:
// the EU profile, UK extension, and the other extensions in extensions.json.
package ocds
//go:generate go run git.avsdev.uk/Go/go-ocds/cmd/ocdsgen generate -c extensions.json -o ocds_gen.go -p ocds
scripts/build.sh
+3
View File
@@ -39,6 +39,9 @@ if [ "$GIT_COMMIT" != "" ]; then
LD_FLAGS="$LD_FLAGS -X git.avsdev.uk/Go/JobSearch/internal/version.GitCommit=${GIT_COMMIT}"
fi
# Process any generators
go generate ./...
# Build binary with version info
go build \
-ldflags="$LD_FLAGS" \