package fetcher import ( "context" "errors" "fmt" "log/slog" "sync" "time" "git.avsdev.uk/Go/job-search/internal/config" "git.avsdev.uk/Go/job-search/internal/store" "git.avsdev.uk/Go/job-search/internal/types" "git.avsdev.uk/Go/job-search/internal/util" ) // ErrNoNewReleases is returned by a source's task when there was nothing new // to fetch; it wraps util.ErrWorkerNoWork. var ErrNoNewReleases = fmt.Errorf("%w: no new releases", util.ErrWorkerNoWork) // Option configures a Fetcher. type Option func(*Fetcher) // WithLogger makes Run, its workers and their clients log to logger instead // of slog.Default(). func WithLogger(logger *slog.Logger) Option { return func(f *Fetcher) { f.log = logger } } // WithStore makes the workers save the opportunities they fetch to // opportunityStore. Without it, nothing is persisted. func WithStore(opportunityStore *store.Store) Option { return func(f *Fetcher) { f.store = opportunityStore } } // saveFunc saves the opportunities a worker fetched; see Fetcher.save. type saveFunc func( ctx context.Context, logger *slog.Logger, opportunities []*types.Opportunity, ) error // logger returns the logger Run logs to: the given one, or slog.Default(). // It isn't tagged here, as it is also given to the clients the workers // create, which add their own component; see tagged. func (f *Fetcher) logger() *slog.Logger { if f.log == nil { return slog.Default() } return f.log } // tagged returns logger tagged with the fetcher's component, for the lines // Run and its workers log themselves. func tagged(logger *slog.Logger) *slog.Logger { return logger.With("component", "fetcher") } // Fetcher runs a worker for each enabled source. Create it with New and // start it with Run. type Fetcher struct { conf *config.JSCSources log *slog.Logger store *store.Store // nil: nothing is persisted started time.Time // backgroundCtx is the context the workers run under. backgroundCancel // stops them: Run calls it when its ctx is cancelled, when a worker // fails, and as it returns. backgroundCtx context.Context backgroundCancel context.CancelFunc } // New returns a Fetcher for the sources conf configures. It logs to // slog.Default() unless given WithLogger. func New(conf *config.JSCSources, opts ...Option) *Fetcher { fetcher := &Fetcher{ conf: conf, started: time.Now(), } for _, option := range opts { option(fetcher) } fetcher.backgroundCtx, fetcher.backgroundCancel = context.WithCancel( context.Background(), ) return fetcher } // Run starts a worker for each enabled source and blocks until every worker // has stopped. Workers stop when ctx is cancelled, or when any worker stops // with an error, which starts a shutdown of all of them. The errors are // logged as they happen and returned joined; nil means a clean shutdown. Run // cancels the Fetcher's background context as it returns, so it can only be // called once. func (f *Fetcher) Run(ctx context.Context, since time.Time) error { defer f.backgroundCancel() // Cancelling ctx stops the workers, through the background context. stopAfter := context.AfterFunc(ctx, f.backgroundCancel) defer stopAfter() logger := f.logger() var ( wg sync.WaitGroup mu sync.Mutex errs []error ) // start runs one source's worker until f.backgroundCtx is cancelled. Each // worker is given logger untagged: it tags its own lines with tagged, and // passes logger as it is to any client it creates. start := func(name string, run func(ctx context.Context) error) { wg.Add(1) go func() { defer wg.Done() if err := run(f.backgroundCtx); err != nil { tagged(logger).Error( "source stopped, shutting down", "source", name, "error", err, ) mu.Lock() errs = append(errs, fmt.Errorf("%s: %w", name, err)) mu.Unlock() f.backgroundCancel() } }() } if f.conf.FindATender.Enabled { start("FindATender", func(ctx context.Context) error { return runFindATender(ctx, logger, &f.conf.FindATender, f.save, since) }) } if f.conf.FTSMock.Enabled { start("FTSMock", func(ctx context.Context) error { return runFTSMock(ctx, logger, &f.conf.FTSMock, f.save, since) }) } if f.conf.Sleeper.Enabled { start("Sleeper", func(ctx context.Context) error { return runSleeper(ctx, logger, &f.conf.Sleeper, since) }) } // Every worker has stopped once Wait returns, so errs is no longer // written. wg.Wait() return errors.Join(errs...) } // save saves opportunities to the store given WithStore, and logs what // changed. Without a store it does nothing. A failure fails the worker's run, // so its checkpoint isn't moved on and the next run fetches them again. func (f *Fetcher) save( ctx context.Context, logger *slog.Logger, opportunities []*types.Opportunity, ) error { if f.store == nil { return nil } result, err := f.store.SaveOpportunities(ctx, opportunities) if err != nil { return fmt.Errorf("saving opportunities: %w", err) } tagged(logger).Info( "saved opportunities", "new_opportunities", result.NewOpportunities, "updated_opportunities", result.UpdatedOpportunities, "new_releases", result.NewReleases, ) return nil }