diff --git a/README.md b/README.md index 2e3c28d..eb44e49 100644 --- a/README.md +++ b/README.md @@ -2,3 +2,17 @@ JobSearch is a service for scraping for contract opportunities. It is primarily a re-write of the existing nodeJS JobSearch codebase. + + +## Installation of dev tools +A number of tools have been used in the development of the project. Installation +of these tools can be done using `go install`. + +To install the tools: +```bash +go install github.com/spf13/cobra-cli@latest +go install git.avsdev.uk/Go/go-ocds/cmd/ocdsgen@latest +``` + +Ensure that the GOBIN path ($HOME/go/bin) is added to the environment before +running any of the tools. diff --git a/cmd/jobsearch/main.go b/cmd/jobsearch/main.go index f2b71e4..67085de 100644 --- a/cmd/jobsearch/main.go +++ b/cmd/jobsearch/main.go @@ -26,6 +26,7 @@ import ( "git.avsdev.uk/Go/job-search/internal/cli" "git.avsdev.uk/Go/job-search/internal/cli/config" + "git.avsdev.uk/Go/job-search/internal/cli/fetch" "git.avsdev.uk/Go/job-search/internal/cli/version" "github.com/spf13/cobra" ) @@ -72,6 +73,7 @@ opportunities. root.AddCommand( version.NewCommand(), config.NewCommand(opts), + fetch.NewCommand(opts), ) return root diff --git a/go.mod b/go.mod index 13061ef..efa569d 100644 --- a/go.mod +++ b/go.mod @@ -3,15 +3,17 @@ module git.avsdev.uk/Go/job-search go 1.26.8 require ( + git.avsdev.uk/Go/go-ocds v1.0.0 github.com/joho/godotenv v1.5.1 github.com/lmittmann/tint v1.2.0 github.com/spf13/cobra v1.10.2 github.com/spf13/viper v1.21.0 go.yaml.in/yaml/v3 v3.0.5 + resty.dev/v3 v3.0.0-rc.4 ) require ( - github.com/fsnotify/fsnotify v1.9.0 // indirect + github.com/fsnotify/fsnotify v1.10.1 // indirect github.com/go-viper/mapstructure/v2 v2.4.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/pelletier/go-toml/v2 v2.2.4 // indirect @@ -20,7 +22,9 @@ require ( github.com/spf13/afero v1.15.0 // indirect github.com/spf13/cast v1.10.0 // indirect github.com/spf13/pflag v1.0.10 // indirect + github.com/stretchr/testify v1.12.1 // indirect github.com/subosito/gotenv v1.6.0 // indirect - golang.org/x/sys v0.29.0 // indirect - golang.org/x/text v0.28.0 // indirect + golang.org/x/net v0.58.0 // indirect + golang.org/x/sys v0.47.0 // indirect + golang.org/x/text v0.41.0 // indirect ) diff --git a/go.sum b/go.sum index 9cc21fd..6d7ed93 100644 --- a/go.sum +++ b/go.sum @@ -1,16 +1,28 @@ +git.avsdev.uk/Go/go-ocds v1.0.0 h1:X/CoCyDcVy8ObGZ+UK4t6viPGj2bqATTEYc4Q6/2DKA= +git.avsdev.uk/Go/go-ocds v1.0.0/go.mod h1:YPnqdf7ft6vPHiHLzeL/R7RtWYNKZJF1NXu/YEBpEbA= github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= -github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k= -github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= +github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= +github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= +github.com/fsnotify/fsnotify v1.10.1 h1:b0/UzAf9yR5rhf3RPm9gf3ehBPpf0oZKIjtpKrx59Ho= +github.com/fsnotify/fsnotify v1.10.1/go.mod h1:TLheqan6HD6GBK6PrDWyDPBaEV8LspOxvPSjC+bVfgo= github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9LvH92wZUgs= github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/lmittmann/tint v1.2.0 h1:AogHRHy8HUJUnNJBHJlYa+fR4YY8mko2cnCp67xn9JY= github.com/lmittmann/tint v1.2.0/go.mod h1:HIS3gSy7qNwGCj+5oRjAutErFBl4BzdQP6cJZ0NfMwE= github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4= github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= +github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= +github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sagikazarmark/locafero v0.11.0 h1:1iurJgmM9G3PA/I+wWYIOw/5SyBtxapeHDcg+AAIFXc= github.com/sagikazarmark/locafero v0.11.0/go.mod h1:nVIGvgyzw595SUSUE6tvCp3YYTeHs15MvlmU87WwIik= @@ -22,19 +34,24 @@ github.com/spf13/cast v1.10.0 h1:h2x0u2shc1QuLHfxi+cTJvs30+ZAHOGRic8uyGTDWxY= github.com/spf13/cast v1.10.0/go.mod h1:jNfB8QC9IA6ZuY2ZjDp0KtFO2LZZlg4S/7bzP6qqeHo= github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU= github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4= -github.com/spf13/pflag v1.0.9 h1:9exaQaMOCwffKiiiYk6/BndUBv+iRViNW+4lEMi0PvY= github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk= github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.21.0 h1:x5S+0EU27Lbphp4UKm1C+1oQO+rKx36vfCoaVebLFSU= github.com/spf13/viper v1.21.0/go.mod h1:P0lhsswPGWD/1lZJ9ny3fYnVqxiegrlNrEmgLjbTCAY= +github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE= +github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg= github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw= go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg= -golang.org/x/sys v0.29.0 h1:TPYlXGxvx1MGTn2GiZDhnjPA9wZzZeGKHHmKhHYvgaU= -golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/text v0.28.0 h1:rhazDwis8INMIwQ4tpjLDzUhx6RlXqZNPEM0huQojng= -golang.org/x/text v0.28.0/go.mod h1:U8nCwOR8jO/marOQ0QbDiOngZVEBB7MAiitBuMjXiNU= +golang.org/x/net v0.58.0 h1:ynWG7rqYi4ccpTEuPZ2QGWHktVEM9DMCj9yzDE0Q7To= +golang.org/x/net v0.58.0/go.mod h1:YwCddHnFlT7eLQqVprV19OnhLGtc5xOKgE0RyqgfWAU= +golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= +golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/text v0.41.0 h1:vz/seA0lnX87Othu2f/0L24RcgrXD9/YFTSuGjj3rH8= +golang.org/x/text v0.41.0/go.mod h1:jvf1O8ajNzZqhSrQBPbutR/EB83Cc0CFrezNQIwbb5M= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +resty.dev/v3 v3.0.0-rc.4 h1:n3360yEcFiyrX5Vkc9XG3Y8kMM24alRBxJ9Ev1mLmqs= +resty.dev/v3 v3.0.0-rc.4/go.mod h1:NTOerrC/4T7/FE6tXIZGIysXXBdgNqwMZuKtxpea9NM= diff --git a/internal/cli/fetch/command.go b/internal/cli/fetch/command.go new file mode 100644 index 0000000..904202b --- /dev/null +++ b/internal/cli/fetch/command.go @@ -0,0 +1,39 @@ +/* +Copyright © 2026 Craig Williams + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Lesser General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Lesser General Public License for more details. + +You should have received a copy of the GNU Lesser General Public License +along with this program. If not, see . +*/ +package fetch + +import ( + "log/slog" + + "git.avsdev.uk/Go/job-search/internal/cli" + "github.com/spf13/cobra" +) + +func NewCommand(opts *cli.Options) *cobra.Command { + cmd := &cobra.Command{ + Use: "fetch", + Short: "Fetch current opportunities", + PersistentPreRun: func(cmd *cobra.Command, args []string) { + slog.Debug("fetch.PersistentPreRun") + }, + PersistentPostRun: func(cmd *cobra.Command, args []string) { + slog.Debug("fetch.PersistentPostRun") + }, + } + cmd.AddCommand(commandOnce(opts)) + return cmd +} diff --git a/internal/cli/fetch/once.go b/internal/cli/fetch/once.go new file mode 100644 index 0000000..4547f5c --- /dev/null +++ b/internal/cli/fetch/once.go @@ -0,0 +1,132 @@ +/* +Copyright © 2026 Craig Williams + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Lesser General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Lesser General Public License for more details. + +You should have received a copy of the GNU Lesser General Public License +along with this program. If not, see . +*/ +package fetch + +import ( + "encoding/json" + "fmt" + "log/slog" + "slices" + "time" + + "git.avsdev.uk/Go/job-search/internal/cli" + "git.avsdev.uk/Go/job-search/internal/findATender" + "git.avsdev.uk/Go/job-search/internal/findATender/ocds" + "github.com/spf13/cobra" +) + +func filter(reqs []ocds.Release, tag string) []ocds.Release { + var resp []ocds.Release + for _, req := range reqs { + if slices.Contains(req.Tag, tag) { + resp = append(resp, req) + } + } + return resp +} + +func commandOnce(opts *cli.Options) *cobra.Command { + cmd := &cobra.Command{ + Use: "once", + Short: "Fetch current opportunities - single shot", + PersistentPreRun: func(cmd *cobra.Command, args []string) { + slog.Debug("fetch.once.PersistentPreRun") + }, + PersistentPostRun: func(cmd *cobra.Command, args []string) { + slog.Debug("fetch.once.PersistentPostRun") + }, + RunE: func(cmd *cobra.Command, args []string) error { + slog.Debug("fetch.once.Run") + + conf, err := opts.RunSetup(cmd, true, true) + if err != nil { + return fmt.Errorf("no configuration loaded (%w)", err) + } + + since, sinceErr := cmd.Flags().GetTime("since") + if sinceErr != nil { + return sinceErr + } + + ft := findATender.New(conf, conf.Sources.FindATender.APIRoot) + defer ft.Shutdown() + + ignoreRateLimit, _ := cmd.Flags().GetBool("ignore-rate-limit") + + results, err := ft.FetchAll(since, !ignoreRateLimit) + if err != nil { + return err + } + + planning, _ := cmd.Flags().GetBool("planning") + opportunities, _ := cmd.Flags().GetBool("opportunities") + awards, _ := cmd.Flags().GetBool("awards") + + if (planning && (opportunities || awards)) || + (opportunities && (planning || awards)) || + (awards && (planning || opportunities)) { + return fmt.Errorf( + "--planning, --opportunities and --awards are exclusive, only one can be provided", + ) + } + + filterTag := "" + if planning { + filterTag = "planning" + } + if opportunities { + filterTag = "opportunity" + } + if awards { + filterTag = "award" + } + + if len(filterTag) > 0 { + results = filter(results, filterTag) + } + + jbytes, err := json.MarshalIndent(results, "", " ") + if err != nil { + return err + } + fmt.Println(string(jbytes)) + + return nil + }, + } + cmd.Flags().Time( + "since", + time.Time{}, + []string{ + time.DateOnly, time.DateTime, + }, + "contract updates published since YYYY-MM-DD[ hh:mm:ss]", + ) + cmd.Flags().Bool( + "planning", false, "only return planning (i.e. exclude opportunities and awards)", + ) + cmd.Flags().Bool( + "opportunities", false, "only return opportunities (i.e. exclude awards and planning)", + ) + cmd.Flags().Bool( + "awards", false, "only return awards (i.e. exclude planning and opportunities)", + ) + cmd.Flags().Bool( + "ignore-rate-limit", false, "ignore the rate limit and fail if exceeded", + ) + return cmd +} diff --git a/internal/config/conf_source_findTender.go b/internal/config/conf_source_findTender.go new file mode 100644 index 0000000..42dce63 --- /dev/null +++ b/internal/config/conf_source_findTender.go @@ -0,0 +1,48 @@ +/* +Copyright © 2026 Craig Williams + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Lesser General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Lesser General Public License for more details. + +You should have received a copy of the GNU Lesser General Public License +along with this program. If not, see . +*/ +package config + +import "reflect" + +const JSCKSourceFindATender string = "find_a_tender" + +type JobSearchConfigSourceFindATender struct { + JobSearchConfigSource + APIRoot string `cfg:"api_root" default:"https://www.find-tender.service.gov.uk/api/1.0"` +} + +/* + * General section functions + * See related methods in section.go + */ +func (c JobSearchConfigSourceFindATender) Map() map[string]any { return mapOf(c) } + +func (c JobSearchConfigSourceFindATender) Flatten() map[string]any { + return flattenOf(JSCKSourceFindATender, c) +} + +func (c JobSearchConfigSourceFindATender) TypeOf(key string) (reflect.Kind, error) { + return typeOf(c, key) +} + +func (c JobSearchConfigSourceFindATender) ValueOf(key string) (any, error) { + return valueOf(c, key) +} + +func (c *JobSearchConfigSourceFindATender) SetValue(key string, value any) error { + return setValue(c, key, value) +} diff --git a/internal/config/conf_sources.go b/internal/config/conf_sources.go new file mode 100644 index 0000000..ce44263 --- /dev/null +++ b/internal/config/conf_sources.go @@ -0,0 +1,83 @@ +/* +Copyright © 2026 Craig Williams + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Lesser General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Lesser General Public License for more details. + +You should have received a copy of the GNU Lesser General Public License +along with this program. If not, see . +*/ +package config + +import ( + "reflect" + "time" +) + +const JSCKSources string = "sources" + +// Each field's cfg tag must match the section's JSCK* constant. +type JobSearchConfigSources struct { + FindATender JobSearchConfigSourceFindATender `cfg:"find_a_tender"` +} + +/* + * General section functions + * See related methods in section.go + */ +func (c JobSearchConfigSources) Map() map[string]any { return mapOf(c) } + +func (c JobSearchConfigSources) Flatten() map[string]any { + return flattenOf(JSCKSources, c) +} + +func (c JobSearchConfigSources) TypeOf(key string) (reflect.Kind, error) { + return typeOf(c, key) +} + +func (c JobSearchConfigSources) ValueOf(key string) (any, error) { + return valueOf(c, key) +} + +func (c *JobSearchConfigSources) SetValue(key string, value any) error { + return setValue(c, key, value) +} + +/* + *** Generic source definition, embedded in each source *** + */ +type JobSearchConfigSource struct { + Enabled bool `cfg:"enabled"` + Interval time.Duration `cfg:"interval" default:"10m"` +} + +/* + * General section functions + * See related methods in section.go + */ +func (c *JobSearchConfigSource) Keys() []string { return keysOf(c) } + +func (c JobSearchConfigSource) Map() map[string]any { return mapOf(c) } + +func (c JobSearchConfigSource) Flatten() map[string]any { + return flattenOf("", c) +} + +func (c JobSearchConfigSource) TypeOf(key string) (reflect.Kind, error) { + return typeOf(c, key) +} + +func (c JobSearchConfigSource) ValueOf(key string) (any, error) { + return valueOf(c, key) +} + +func (c *JobSearchConfigSource) SetValue(key string, value any) error { + return setValue(c, key, value) +} diff --git a/internal/config/config.go b/internal/config/config.go index 5123a72..ae44520 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -32,7 +32,8 @@ const EtcConfigPath string = "/etc/job-search" // Each field's cfg tag must match the section's JSCK* constant. type JobSearchConfig struct { - Logging JobSearchConfigLogging `cfg:"logging"` + Logging JobSearchConfigLogging `cfg:"logging"` + Sources JobSearchConfigSources `cfg:"sources"` } // Default configuration, taken from the default struct tags diff --git a/internal/findATender/.gitignore b/internal/findATender/.gitignore new file mode 100644 index 0000000..b89ef5d --- /dev/null +++ b/internal/findATender/.gitignore @@ -0,0 +1 @@ +ocds_gen.go \ No newline at end of file diff --git a/internal/findATender/api.go b/internal/findATender/api.go new file mode 100644 index 0000000..6f64eee --- /dev/null +++ b/internal/findATender/api.go @@ -0,0 +1,218 @@ +/* +Copyright © 2026 Craig Williams + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Lesser General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Lesser General Public License for more details. + +You should have received a copy of the GNU Lesser General Public License +along with this program. If not, see . +*/ +package findATender + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "log/slog" + "strconv" + "time" + + "git.avsdev.uk/Go/job-search/internal/findATender/ocds" + "git.avsdev.uk/Go/job-search/internal/logging" + resty "resty.dev/v3" +) + +var ErrNoNext = errors.New("no next link") +var ErrNoReleases = errors.New("no releases") +var ErrRateLimitExceeded = errors.New("rate limit has been exceeded") +var ErrFetchCancelled = errors.New("fetch cancelled") + +type RequestOptions struct { + Since time.Time + Limit int + AllowRateLimit bool +} + +func rateSleep(ctx context.Context, duration time.Duration) error { + timer := time.NewTimer(duration) + defer timer.Stop() + + select { + case <-timer.C: + return nil + case <-ctx.Done(): + return ctx.Err() + } +} + +func fetchURI(ft *FindATender, uri string) (*resty.Response, error) { + withTrace := ft.config.Logging.LogLevel() <= slog.LevelDebug + + slog.Info("findTender.resty", "request_uri", uri) + + ctx := ft.ctx + if ctx == nil { + ctx = context.Background() + } + + req := ft.httpClient.R() + req.SetContext(ctx) + res, err := req.SetTrace(withTrace).Get(uri) + + if withTrace { + if logging.IsJSONHandler(slog.Default().Handler()) { + slog.Debug( + "Resty", + "trace", res.Request.TraceInfo().JSON(), + "status", res.Status(), + "response", res, + ) + } else { + slog.Debug("Resty " + res.Request.TraceInfo().String()) + slog.Debug("Resty status: " + res.Status()) + slog.Debug("Resty response: " + res.String()) + } + } + + return res, err +} + +func (ft *FindATender) fetchURI( + uri string, + allowRateLimit bool, +) (*resty.Response, error) { + res, err := fetchURI(ft, uri) + if err != nil { + return nil, err + } + + if ft.ctx == nil { + copied := *ft + copied.ctx = context.Background() + ft = &copied + } + + if res.StatusCode() == 429 { + if !allowRateLimit { + return nil, ErrRateLimitExceeded + } + + retrySecStr := res.Header().Get("Retry-After") + retrySec, err := strconv.ParseInt(retrySecStr, 10, 32) + if err != nil { + return nil, fmt.Errorf("unknown retry-after value: %w", err) + } + + slog.Warn(fmt.Sprintf( + "API rate limit hit, waiting as requested for %d seconds", retrySec, + )) + if rateSleep(ft.ctx, time.Duration(retrySec)*time.Second) != nil { + return nil, ErrFetchCancelled + } + + res, err = fetchURI(ft, uri) + if err != nil { + return nil, err + } + } + + return res, nil +} + +func (ft *FindATender) fetchWithOpts( + opts RequestOptions, +) (*resty.Response, error) { + if opts.Since.IsZero() { + opts.Since = time.Now().AddDate(0, 0, 1) + } + if opts.Limit == 0 { + opts.Limit = 100 + } + + uri := fmt.Sprintf( + "%s/ocdsReleasePackages?updatedFrom=%s&limit=%d", + ft.apiRoot, + opts.Since.Format("2006-01-02T15:04:05"), + opts.Limit, + ) + + return ft.fetchURI(uri, opts.AllowRateLimit) +} + +func (ft *FindATender) fetchNext( + pkg ocds.ReleasePackage, + allowRateLimit bool, +) (*resty.Response, error) { + if pkg.Links == nil || len(pkg.Links.Next) == 0 { + return nil, ErrNoNext + } + return ft.fetchURI(pkg.Links.Next, allowRateLimit) +} + +func (ft *FindATender) fetchOnce(opts RequestOptions) ([]ocds.Release, error) { + res, err := ft.fetchWithOpts(opts) + if err != nil { + return nil, err + } + defer res.Body.Close() + + var relPkg ocds.ReleasePackage + decoder := json.NewDecoder(NewOCDSStreamCleaner(res.Body)) + if err := decoder.Decode(&relPkg); err != nil { + return nil, err + } + + if len(relPkg.Releases) == 0 { + return nil, ErrNoReleases + } + + return relPkg.Releases, nil +} + +func (ft *FindATender) fetchAll(opts RequestOptions) ([]ocds.Release, error) { + var releases []ocds.Release + + res, err := ft.fetchWithOpts(opts) + if err != nil { + return nil, err + } + defer res.Body.Close() + + var relPkg ocds.ReleasePackage + decoder := json.NewDecoder(NewOCDSStreamCleaner(res.Body)) + if err := decoder.Decode(&relPkg); err != nil { + return nil, err + } + releases = append(releases, relPkg.Releases...) + + for true { + res, err := ft.fetchNext(relPkg, opts.AllowRateLimit) + if err == ErrNoNext { + break + } + if err != nil { + return nil, err + } + defer res.Body.Close() + + decoder := json.NewDecoder(NewOCDSStreamCleaner(res.Body)) + if err := decoder.Decode(&relPkg); err != nil { + return nil, err + } + releases = append(releases, relPkg.Releases...) + } + + if len(releases) == 0 { + return nil, ErrNoReleases + } + + return releases, nil +} diff --git a/internal/findATender/dateRepair.go b/internal/findATender/dateRepair.go new file mode 100644 index 0000000..eaa70c4 --- /dev/null +++ b/internal/findATender/dateRepair.go @@ -0,0 +1,59 @@ +/* +Copyright © 2026 Craig Williams + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Lesser General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Lesser General Public License for more details. + +You should have received a copy of the GNU Lesser General Public License +along with this program. If not, see . +*/ +package findATender + +import ( + "bufio" + "bytes" + "io" + "regexp" +) + +// Global regex to identify and catch date-only fields +var invalidDateRegex = regexp.MustCompile(`:\s*"(\d{4}-\d{2}-\d{2})"\s*([,}\s])`) +var invalidPublishedDateRegex = regexp.MustCompile(`"publishedDate":\s*""`) + +// The UK Government FindATender service doesn't strictly follow OCDS date +// format. Sometimes it provides dates only, sometimes it sends empty strings +// instead of nulls. +type FindATenderStreamCleaner struct { + reader *bufio.Reader + buffer bytes.Buffer +} + +func NewOCDSStreamCleaner(r io.Reader) *FindATenderStreamCleaner { + return &FindATenderStreamCleaner{reader: bufio.NewReader(r)} +} + +func (sc *FindATenderStreamCleaner) Read(p []byte) (int, error) { + if sc.buffer.Len() == 0 { + line, err := sc.reader.ReadBytes('\n') + if len(line) > 0 { + // Convert "2026-11-16" -> "2026-11-16T00:00:00Z" + line = invalidDateRegex.ReplaceAll(line, []byte(`:"$1T00:00:00Z"`)) + // Convert {publishedDate:""} -> "{publishedDate:null}" + line = invalidPublishedDateRegex.ReplaceAll(line, []byte(`"publishedDate":null`)) + sc.buffer.Write(line) + } + if err != nil { + if sc.buffer.Len() == 0 { + return 0, err + } + } + } + return sc.buffer.Read(p) +} diff --git a/internal/findATender/findATender.go b/internal/findATender/findATender.go new file mode 100644 index 0000000..ca2d32d --- /dev/null +++ b/internal/findATender/findATender.go @@ -0,0 +1,91 @@ +/* +Copyright © 2026 Craig Williams + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Lesser General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Lesser General Public License for more details. + +You should have received a copy of the GNU Lesser General Public License +along with this program. If not, see . +*/ +package findATender + +import ( + "context" + "time" + + "git.avsdev.uk/Go/job-search/internal/config" + "git.avsdev.uk/Go/job-search/internal/findATender/ocds" + resty "resty.dev/v3" +) + +type FindATender struct { + httpClient *resty.Client + config *config.JobSearchConfig + apiRoot string + ctx context.Context +} + +func New(conf *config.JobSearchConfig, apiRoot string) FindATender { + ft := FindATender{ + config: conf, + apiRoot: apiRoot, + } + ft.httpClient = resty.New(). + SetTimeout(60 * time.Duration(time.Second)). + SetRetryCount(1) + return ft +} + +func (ft *FindATender) ReloadConfig(conf *config.JobSearchConfig) { + ft.config = conf + // TODO: any reloading required for configuration change handling +} + +func (ft *FindATender) With(ctx context.Context) *FindATender { + if ctx == nil { + return ft + } + + copied := *ft + copied.ctx = ctx + return &copied +} + +func (ft *FindATender) Shutdown() { + ft.httpClient.Close() +} + +func (ft *FindATender) FetchOnce( + since time.Time, + allowRateLimit bool, +) ([]ocds.Release, error) { + reqs, err := ft.fetchOnce(RequestOptions{ + Since: since, + AllowRateLimit: allowRateLimit, + }) + if err != nil { + return nil, err + } + return reqs, nil +} + +func (ft *FindATender) FetchAll( + since time.Time, + allowRateLimit bool, +) ([]ocds.Release, error) { + reqs, err := ft.fetchAll(RequestOptions{ + Since: since, + AllowRateLimit: allowRateLimit, + }) + if err != nil { + return nil, err + } + return reqs, nil +} diff --git a/internal/findATender/ocds/extensions.json b/internal/findATender/ocds/extensions.json new file mode 100644 index 0000000..ba38ac6 --- /dev/null +++ b/internal/findATender/ocds/extensions.json @@ -0,0 +1,16 @@ +{ + "releaseSchema": "https://raw.githubusercontent.com/open-contracting/standard/1.1/schema/release-schema.json", + "releasePackageSchema": "https://raw.githubusercontent.com/open-contracting/standard/1.1/schema/release-package-schema.json", + "extensions": [ + "https://standard.open-contracting.org/profiles/eu/latest/en/extension.json", + "https://raw.githubusercontent.com/open-contracting-extensions/ocds_amendment_rationale_classifications_extension/master/extension.json", + "https://raw.githubusercontent.com/open-contracting-extensions/ocds_budget_breakdown_extension/master/extension.json", + "https://raw.githubusercontent.com/open-contracting-extensions/ocds_contract_completion_extension/master/extension.json", + "https://raw.githubusercontent.com/open-contracting-extensions/ocds_documentation_extension/master/extension.json", + "https://raw.githubusercontent.com/open-contracting-extensions/ocds_pagination_extension/master/extension.json", + "https://raw.githubusercontent.com/open-contracting-extensions/ocds_suitability_extension/master/extension.json", + "https://raw.githubusercontent.com/portaledcahn/ocds_Links_extension/main/extension.json", + "https://raw.githubusercontent.com/cabinetoffice/ocds_uk_extension/main/extension.json", + "https://raw.githubusercontent.com/open-contracting-extensions/ocds_performance_failures_extension/master/extension.json" + ] +} diff --git a/internal/findATender/ocds/generate.go b/internal/findATender/ocds/generate.go new file mode 100644 index 0000000..25b431a --- /dev/null +++ b/internal/findATender/ocds/generate.go @@ -0,0 +1,5 @@ +// Package full contains OCDS 1.1 types with the full extension set applied: +// the EU profile, UK extension, and the other extensions in extensions.json. +package ocds + +//go:generate go run git.avsdev.uk/Go/go-ocds/cmd/ocdsgen generate -c extensions.json -o ocds_gen.go -p ocds diff --git a/scripts/build.sh b/scripts/build.sh index f3f281d..77c6811 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -39,6 +39,9 @@ if [ "$GIT_COMMIT" != "" ]; then LD_FLAGS="$LD_FLAGS -X git.avsdev.uk/Go/JobSearch/internal/version.GitCommit=${GIT_COMMIT}" fi +# Process any generators +go generate ./... + # Build binary with version info go build \ -ldflags="$LD_FLAGS" \