fetcher.Run(ctx, conf, since) becomes New(&conf.Sources, opts...) and Run(ctx, since). Each worker now takes only its own source's config, and Run starts each one through a closure that binds it. Cancelling ctx stops the workers through the Fetcher's background context, and Run always waits for every worker before returning their joined errors. The findatender client no longer holds the configuration: New takes only the API root, ReloadConfig is gone, and request tracing follows logging.IsDebugEnabled(). The FindAOpportunity worker is now FindATender. job-search serve runs the fetcher alongside the HTTP server. A source that fails stops the server, and the server stopping stops the sources; with no sources enabled the server keeps running. fetcher service and fetcher fts-once use the new APIs.
150 lines
3.8 KiB
Go
150 lines
3.8 KiB
Go
package fetch
|
|
|
|
import (
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"log/slog"
|
|
"slices"
|
|
"time"
|
|
|
|
"github.com/spf13/cobra"
|
|
|
|
"git.avsdev.uk/Go/job-search/internal/cli"
|
|
"git.avsdev.uk/Go/job-search/internal/findatender"
|
|
"git.avsdev.uk/Go/job-search/internal/types"
|
|
)
|
|
|
|
// Release tags for each stage of a procedure, from the OCDS releaseTag
|
|
// codelist. Cancellations are left out, as a cancelled procedure is no longer
|
|
// at any stage.
|
|
var stageTags = map[string][]string{
|
|
"planning": {"planning", "planningUpdate"},
|
|
"opportunity": {"tender", "tenderAmendment", "tenderUpdate"},
|
|
"award": {
|
|
"award", "awardUpdate",
|
|
"contract", "contractUpdate", "contractAmendment",
|
|
"implementation", "implementationUpdate", "contractTermination",
|
|
},
|
|
}
|
|
|
|
// filter keeps the opportunities whose latest release is at stage. Releases
|
|
// are oldest first, so the last one is the procedure's current stage.
|
|
func filter(
|
|
opportunities []*types.Opportunity, stage string,
|
|
) []*types.Opportunity {
|
|
var matching []*types.Opportunity
|
|
|
|
for _, opportunity := range opportunities {
|
|
if len(opportunity.Releases) == 0 {
|
|
continue
|
|
}
|
|
latest := opportunity.Releases[len(opportunity.Releases)-1]
|
|
for _, tag := range latest.Tags {
|
|
if slices.Contains(stageTags[stage], tag.Tag) {
|
|
matching = append(matching, opportunity)
|
|
break
|
|
}
|
|
}
|
|
}
|
|
|
|
return matching
|
|
}
|
|
|
|
// commandFTSOnce returns the "fts-once" command, which fetches from Find a
|
|
// Tender once and prints the releases as JSON.
|
|
func commandFTSOnce(opts *cli.Options) *cobra.Command {
|
|
cmd := &cobra.Command{
|
|
Use: "fts-once",
|
|
Short: "Fetch from Find a Tender once and print the releases as JSON",
|
|
PersistentPreRun: func(cmd *cobra.Command, args []string) {
|
|
slog.Debug("command persistent pre-run", "command", "fts-once")
|
|
},
|
|
PersistentPostRun: func(cmd *cobra.Command, args []string) {
|
|
slog.Debug("command persistent post-run", "command", "fts-once")
|
|
},
|
|
RunE: func(cmd *cobra.Command, args []string) error {
|
|
slog.Debug("command run", "command", "fts-once")
|
|
|
|
conf, err := opts.RunSetup(cmd, true, true)
|
|
if err != nil {
|
|
return fmt.Errorf("load configuration: %w", err)
|
|
}
|
|
|
|
since, sinceErr := cmd.Flags().GetTime("since")
|
|
if sinceErr != nil {
|
|
return sinceErr
|
|
}
|
|
|
|
if conf.Sources.FindATender.APIRoot == "" {
|
|
return errors.New("sources.find_a_tender.api_root is not set")
|
|
}
|
|
findATender := findatender.New(conf.Sources.FindATender.APIRoot)
|
|
defer findATender.Shutdown()
|
|
|
|
ignoreRateLimit, _ := cmd.Flags().GetBool("ignore-rate-limit")
|
|
|
|
results, err := findATender.FetchAll(since, !ignoreRateLimit)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
planning, _ := cmd.Flags().GetBool("planning")
|
|
opportunities, _ := cmd.Flags().GetBool("opportunities")
|
|
awards, _ := cmd.Flags().GetBool("awards")
|
|
|
|
filterTag := ""
|
|
if planning {
|
|
filterTag = "planning"
|
|
}
|
|
if opportunities {
|
|
filterTag = "opportunity"
|
|
}
|
|
if awards {
|
|
filterTag = "award"
|
|
}
|
|
|
|
if filterTag != "" {
|
|
results = filter(results, filterTag)
|
|
}
|
|
|
|
encoded, err := json.MarshalIndent(results, "", " ")
|
|
if err != nil {
|
|
return err
|
|
}
|
|
fmt.Println(string(encoded))
|
|
|
|
return nil
|
|
},
|
|
}
|
|
cmd.Flags().Time(
|
|
"since",
|
|
time.Time{},
|
|
[]string{
|
|
time.DateOnly, time.DateTime,
|
|
},
|
|
"contract updates published since YYYY-MM-DD[ hh:mm:ss]",
|
|
)
|
|
cmd.Flags().Bool(
|
|
"planning",
|
|
false,
|
|
"only return planning (i.e. exclude opportunities and awards)",
|
|
)
|
|
cmd.Flags().Bool(
|
|
"opportunities",
|
|
false,
|
|
"only return opportunities (i.e. exclude awards and planning)",
|
|
)
|
|
cmd.Flags().Bool(
|
|
"awards",
|
|
false,
|
|
"only return awards (i.e. exclude planning and opportunities)",
|
|
)
|
|
cmd.Flags().Bool(
|
|
"ignore-rate-limit", false, "ignore the rate limit and fail if exceeded",
|
|
)
|
|
cmd.MarkFlagsMutuallyExclusive("planning", "opportunities", "awards")
|
|
|
|
return cmd
|
|
}
|