Files
job-search/internal/cli/fetch/fts_once.go
T
bwilliams 18db462d89 Made the fetcher a Fetcher type with per-source config, and ran it in job-search serve
fetcher.Run(ctx, conf, since) becomes New(&conf.Sources, opts...) and
Run(ctx, since). Each worker now takes only its own source's config, and
Run starts each one through a closure that binds it. Cancelling ctx stops
the workers through the Fetcher's background context, and Run always waits
for every worker before returning their joined errors.

The findatender client no longer holds the configuration: New takes only
the API root, ReloadConfig is gone, and request tracing follows
logging.IsDebugEnabled(). The FindAOpportunity worker is now FindATender.

job-search serve runs the fetcher alongside the HTTP server. A source that
fails stops the server, and the server stopping stops the sources; with no
sources enabled the server keeps running. fetcher service and fetcher
fts-once use the new APIs.
2026-10-16 10:44:38 +01:00

150 lines
3.8 KiB
Go

package fetch
import (
"encoding/json"
"errors"
"fmt"
"log/slog"
"slices"
"time"
"github.com/spf13/cobra"
"git.avsdev.uk/Go/job-search/internal/cli"
"git.avsdev.uk/Go/job-search/internal/findatender"
"git.avsdev.uk/Go/job-search/internal/types"
)
// Release tags for each stage of a procedure, from the OCDS releaseTag
// codelist. Cancellations are left out, as a cancelled procedure is no longer
// at any stage.
var stageTags = map[string][]string{
"planning": {"planning", "planningUpdate"},
"opportunity": {"tender", "tenderAmendment", "tenderUpdate"},
"award": {
"award", "awardUpdate",
"contract", "contractUpdate", "contractAmendment",
"implementation", "implementationUpdate", "contractTermination",
},
}
// filter keeps the opportunities whose latest release is at stage. Releases
// are oldest first, so the last one is the procedure's current stage.
func filter(
opportunities []*types.Opportunity, stage string,
) []*types.Opportunity {
var matching []*types.Opportunity
for _, opportunity := range opportunities {
if len(opportunity.Releases) == 0 {
continue
}
latest := opportunity.Releases[len(opportunity.Releases)-1]
for _, tag := range latest.Tags {
if slices.Contains(stageTags[stage], tag.Tag) {
matching = append(matching, opportunity)
break
}
}
}
return matching
}
// commandFTSOnce returns the "fts-once" command, which fetches from Find a
// Tender once and prints the releases as JSON.
func commandFTSOnce(opts *cli.Options) *cobra.Command {
cmd := &cobra.Command{
Use: "fts-once",
Short: "Fetch from Find a Tender once and print the releases as JSON",
PersistentPreRun: func(cmd *cobra.Command, args []string) {
slog.Debug("command persistent pre-run", "command", "fts-once")
},
PersistentPostRun: func(cmd *cobra.Command, args []string) {
slog.Debug("command persistent post-run", "command", "fts-once")
},
RunE: func(cmd *cobra.Command, args []string) error {
slog.Debug("command run", "command", "fts-once")
conf, err := opts.RunSetup(cmd, true, true)
if err != nil {
return fmt.Errorf("load configuration: %w", err)
}
since, sinceErr := cmd.Flags().GetTime("since")
if sinceErr != nil {
return sinceErr
}
if conf.Sources.FindATender.APIRoot == "" {
return errors.New("sources.find_a_tender.api_root is not set")
}
findATender := findatender.New(conf.Sources.FindATender.APIRoot)
defer findATender.Shutdown()
ignoreRateLimit, _ := cmd.Flags().GetBool("ignore-rate-limit")
results, err := findATender.FetchAll(since, !ignoreRateLimit)
if err != nil {
return err
}
planning, _ := cmd.Flags().GetBool("planning")
opportunities, _ := cmd.Flags().GetBool("opportunities")
awards, _ := cmd.Flags().GetBool("awards")
filterTag := ""
if planning {
filterTag = "planning"
}
if opportunities {
filterTag = "opportunity"
}
if awards {
filterTag = "award"
}
if filterTag != "" {
results = filter(results, filterTag)
}
encoded, err := json.MarshalIndent(results, "", " ")
if err != nil {
return err
}
fmt.Println(string(encoded))
return nil
},
}
cmd.Flags().Time(
"since",
time.Time{},
[]string{
time.DateOnly, time.DateTime,
},
"contract updates published since YYYY-MM-DD[ hh:mm:ss]",
)
cmd.Flags().Bool(
"planning",
false,
"only return planning (i.e. exclude opportunities and awards)",
)
cmd.Flags().Bool(
"opportunities",
false,
"only return opportunities (i.e. exclude awards and planning)",
)
cmd.Flags().Bool(
"awards",
false,
"only return awards (i.e. exclude planning and opportunities)",
)
cmd.Flags().Bool(
"ignore-rate-limit", false, "ignore the rate limit and fail if exceeded",
)
cmd.MarkFlagsMutuallyExclusive("planning", "opportunities", "awards")
return cmd
}