Split fetch from jobsearch into a separate fetcher command, and renamed fetch once to fts-once
job-search no longer has a fetch command. Its subcommands are now in cmd/fetcher, which reads the same configuration: "fetcher service" runs every enabled source's fetcher on its interval until stopped, and "fetcher fts-once" (was "fetch once") fetches from Find a Tender once and prints the releases as JSON. The new namea say what the commands do, because service runs every source while fts-once fetches only from Find a Tender. The code stays in internal/cli/fetch, where NewCommands replaces the fetch parent command. once.go is now fts_once.go and commandOnce is commandFTSOnce. The README's usage, the config README and the find_a_tender api_root comment now give the new command names.
This commit is contained in:
1 parent
5e61c11eb9
commit
ac44b45f06
11 files changed
+155
-44
No files matched your search
@@ -0,0 +1,149 @@
|
||||
package fetch
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"slices"
|
||||
"time"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
|
||||
"git.avsdev.uk/Go/job-search/internal/cli"
|
||||
"git.avsdev.uk/Go/job-search/internal/findatender"
|
||||
"git.avsdev.uk/Go/job-search/internal/types"
|
||||
)
|
||||
|
||||
// Release tags for each stage of a procedure, from the OCDS releaseTag
|
||||
// codelist. Cancellations are left out, as a cancelled procedure is no longer
|
||||
// at any stage.
|
||||
var stageTags = map[string][]string{
|
||||
"planning": {"planning", "planningUpdate"},
|
||||
"opportunity": {"tender", "tenderAmendment", "tenderUpdate"},
|
||||
"award": {
|
||||
"award", "awardUpdate",
|
||||
"contract", "contractUpdate", "contractAmendment",
|
||||
"implementation", "implementationUpdate", "contractTermination",
|
||||
},
|
||||
}
|
||||
|
||||
// filter keeps the opportunities whose latest release is at stage. Releases
|
||||
// are oldest first, so the last one is the procedure's current stage.
|
||||
func filter(
|
||||
opportunities []*types.Opportunity, stage string,
|
||||
) []*types.Opportunity {
|
||||
var matching []*types.Opportunity
|
||||
|
||||
for _, opportunity := range opportunities {
|
||||
if len(opportunity.Releases) == 0 {
|
||||
continue
|
||||
}
|
||||
latest := opportunity.Releases[len(opportunity.Releases)-1]
|
||||
for _, tag := range latest.Tags {
|
||||
if slices.Contains(stageTags[stage], tag.Tag) {
|
||||
matching = append(matching, opportunity)
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return matching
|
||||
}
|
||||
|
||||
// commandFTSOnce returns the "fts-once" command, which fetches from Find a
|
||||
// Tender once and prints the releases as JSON.
|
||||
func commandFTSOnce(opts *cli.Options) *cobra.Command {
|
||||
cmd := &cobra.Command{
|
||||
Use: "fts-once",
|
||||
Short: "Fetch from Find a Tender once and print the releases as JSON",
|
||||
PersistentPreRun: func(cmd *cobra.Command, args []string) {
|
||||
slog.Debug("command persistent pre-run", "command", "fts-once")
|
||||
},
|
||||
PersistentPostRun: func(cmd *cobra.Command, args []string) {
|
||||
slog.Debug("command persistent post-run", "command", "fts-once")
|
||||
},
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
slog.Debug("command run", "command", "fts-once")
|
||||
|
||||
conf, err := opts.RunSetup(cmd, true, true)
|
||||
if err != nil {
|
||||
return fmt.Errorf("load configuration: %w", err)
|
||||
}
|
||||
|
||||
since, sinceErr := cmd.Flags().GetTime("since")
|
||||
if sinceErr != nil {
|
||||
return sinceErr
|
||||
}
|
||||
|
||||
if conf.Sources.FindATender.APIRoot == "" {
|
||||
return errors.New("sources.find_a_tender.api_root is not set")
|
||||
}
|
||||
findATender := findatender.New(conf, conf.Sources.FindATender.APIRoot)
|
||||
defer findATender.Shutdown()
|
||||
|
||||
ignoreRateLimit, _ := cmd.Flags().GetBool("ignore-rate-limit")
|
||||
|
||||
results, err := findATender.FetchAll(since, !ignoreRateLimit)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
planning, _ := cmd.Flags().GetBool("planning")
|
||||
opportunities, _ := cmd.Flags().GetBool("opportunities")
|
||||
awards, _ := cmd.Flags().GetBool("awards")
|
||||
|
||||
filterTag := ""
|
||||
if planning {
|
||||
filterTag = "planning"
|
||||
}
|
||||
if opportunities {
|
||||
filterTag = "opportunity"
|
||||
}
|
||||
if awards {
|
||||
filterTag = "award"
|
||||
}
|
||||
|
||||
if filterTag != "" {
|
||||
results = filter(results, filterTag)
|
||||
}
|
||||
|
||||
encoded, err := json.MarshalIndent(results, "", " ")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Println(string(encoded))
|
||||
|
||||
return nil
|
||||
},
|
||||
}
|
||||
cmd.Flags().Time(
|
||||
"since",
|
||||
time.Time{},
|
||||
[]string{
|
||||
time.DateOnly, time.DateTime,
|
||||
},
|
||||
"contract updates published since YYYY-MM-DD[ hh:mm:ss]",
|
||||
)
|
||||
cmd.Flags().Bool(
|
||||
"planning",
|
||||
false,
|
||||
"only return planning (i.e. exclude opportunities and awards)",
|
||||
)
|
||||
cmd.Flags().Bool(
|
||||
"opportunities",
|
||||
false,
|
||||
"only return opportunities (i.e. exclude awards and planning)",
|
||||
)
|
||||
cmd.Flags().Bool(
|
||||
"awards",
|
||||
false,
|
||||
"only return awards (i.e. exclude planning and opportunities)",
|
||||
)
|
||||
cmd.Flags().Bool(
|
||||
"ignore-rate-limit", false, "ignore the rate limit and fail if exceeded",
|
||||
)
|
||||
cmd.MarkFlagsMutuallyExclusive("planning", "opportunities", "awards")
|
||||
|
||||
return cmd
|
||||
}
|
||||
Reference in new issue
Block a user