bwilliams
5cb3601e1f
Fetching: - Add `fetch service`, which runs a worker per enabled source until SIGINT/SIGTERM (internal/fetcher) - Add util.Worker: runs a task on an interval with a since-checkpoint that only advances on success, and a shutdown timeout for in-flight work; plus AwaitTask/AwaitTaskE for calls that ignore cancellation - Add fts_mock and sleeper sources - Sleeper: sleeps periodically to emulate work being done. Can be instructed to block the shutdown signal or handle it cleanly Mock FTS server (cmd/mock-fts, internal/mock/findATender): - Serves GET /api/1.0/ocdsReleasePackages with FTS paging, query validation, UK local time handling and per-client rate limiting - Generates fictional releases from an embedded catalog, moving procedures through planning, tender and award, and keeps them in a JSON Lines store that survives restarts - Add logging.AccessLog for combined/common access logs
46 lines
870 B
Go
46 lines
870 B
Go
package fetcher
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"log/slog"
|
|
"sync"
|
|
"time"
|
|
|
|
"git.avsdev.uk/Go/job-search/internal/config"
|
|
"git.avsdev.uk/Go/job-search/internal/util"
|
|
)
|
|
|
|
var ErrNoNewReleases = fmt.Errorf("%w: no new releases", util.ErrWorkerNoWork)
|
|
|
|
func Run(
|
|
ctx context.Context, conf *config.JobSearchConfig, since time.Time,
|
|
) {
|
|
var wg sync.WaitGroup
|
|
|
|
start := func(
|
|
name string,
|
|
run func(context.Context, *config.JobSearchConfig, time.Time) error,
|
|
) {
|
|
wg.Add(1)
|
|
go func() {
|
|
defer wg.Done()
|
|
if err := run(ctx, conf, since); err != nil {
|
|
slog.Error("source stopped", "source", name, "err", err)
|
|
}
|
|
}()
|
|
}
|
|
|
|
if conf.Sources.FindATender.Enabled {
|
|
start("FindATender", runFindAOpportunity)
|
|
}
|
|
if conf.Sources.FTSMock.Enabled {
|
|
start("FTSMock", runFTSMock)
|
|
}
|
|
if conf.Sources.Sleeper.Enabled {
|
|
start("Sleeper", runSleeper)
|
|
}
|
|
|
|
wg.Wait()
|
|
}
|