package fetcher import ( "context" "errors" "fmt" "log/slog" "time" "git.avsdev.uk/Go/job-search/internal/config" "git.avsdev.uk/Go/job-search/internal/eusupply" "git.avsdev.uk/Go/job-search/internal/store" "git.avsdev.uk/Go/job-search/internal/types" "git.avsdev.uk/Go/job-search/internal/util" ) // euSupplySource is the Source of the opportunities the EU Supply source // fetches. const euSupplySource = eusupply.Source // repeatWindow is how far apart an EU Supply tender's publication and // another source's opportunity's latest release may be for the two to be // the same (see markRepeats). const repeatWindow = 30 * 24 * time.Hour // tenderStore is what the EU Supply source reads from the store, to skip // tenders already stored and to find the ones another source has. type tenderStore interface { ReleaseExists(ctx context.Context, uuid, releaseID string) (bool, error) ListOtherSourcesTitles( ctx context.Context, source string, from, before time.Time, ) ([]store.OpportunityTitle, error) GetOpportunity(ctx context.Context, uuid string) (*types.Opportunity, error) } // runEUSupply scrapes EU Supply's UK portal every configured interval until // ctx is cancelled, passing what each run fetches to save, even when the run // fails partway. A long window is backfilled first (see backfill). One // client is used throughout, so its gap between requests holds across runs. // lookup is nil without a store: every tender is then new, and none is // checked for repeats. func runEUSupply( ctx context.Context, logger *slog.Logger, conf *config.JSCSourceEUSupply, lookup tenderStore, save saveFunc, since time.Time, ) error { client := eusupply.New( conf.BaseURL, eusupply.WithLogger(logger), eusupply.WithRequestGap(conf.RequestMinGap, conf.RequestMaxGap), ) fetch := func( ctx context.Context, since, until time.Time, ) ([]*types.Opportunity, error) { return fetchEUSupply(ctx, logger, client, lookup, since, until) } since = backfill( ctx, logger, euSupplySource, conf.Interval, fetch, save, since, ) worker := util.Worker{ Logger: tagged(logger), Name: euSupplySource, Interval: conf.Interval, Task: func(ctx context.Context, since time.Time) error { opportunities, err := fetch(ctx, since, time.Time{}) return saveFetched(ctx, logger, save, opportunities, err) }, } return worker.Run(ctx, since) } // fetchEUSupply fetches the tenders published from since up to until (zero: // today) that aren't stored yet, with their own pages read, and marks the // ones another source already has Hidden and Skipped (see markRepeats). If // it fails partway, the tenders read in full before the error are returned // along with it. Finding nothing new returns ErrNoNewReleases. func fetchEUSupply( ctx context.Context, logger *slog.Logger, client *eusupply.Client, lookup tenderStore, since, until time.Time, ) ([]*types.Opportunity, error) { tenders, fetchErr := client.Search(ctx, since, until) fresh, err := newTenders(ctx, lookup, tenders) if err != nil { return nil, err } for i := range fresh { if err := client.Details(ctx, &fresh[i]); err != nil { fresh = fresh[:i] fetchErr = errors.Join(fetchErr, err) break } } opportunities := eusupply.Convert(fresh) if err := markRepeats(ctx, logger, lookup, opportunities); err != nil { // Saved unmarked, a repeat would stay visible, as marks are only // stored for a new opportunity: fetch them again next run instead. return nil, errors.Join(fetchErr, err) } tagged(logger).Info( "fetched opportunities", "source", euSupplySource, "listed", len(tenders), "count", len(opportunities), ) if len(opportunities) == 0 && fetchErr == nil { return nil, ErrNoNewReleases } return opportunities, fetchErr } // newTenders returns the tenders whose release isn't stored yet, so only // their pages are read. Saving a stored one again would replace its row // with the search's fields alone. With no lookup, every tender is new. func newTenders( ctx context.Context, lookup tenderStore, tenders []eusupply.Tender, ) ([]eusupply.Tender, error) { if lookup == nil { return tenders, nil } var fresh []eusupply.Tender for _, tender := range tenders { exists, err := lookup.ReleaseExists( ctx, eusupply.UUID(tender.ID), eusupply.ReleaseID(tender), ) if err != nil { return nil, fmt.Errorf("checking tender %s: %w", tender.ID, err) } if !exists { fresh = append(fresh, tender) } } return fresh, nil } // markRepeats marks Hidden and Skipped each opportunity that repeats one // from another source, most often its Find a Tender notice: its title is at // least titleMatchThreshold alike (see similarTitles), its latest release // is within repeatWindow, and its buyer or description matches too (see // confirmedRepeat). Only the opportunities whose titles match are read in // full. The store keeps the marks of an opportunity it already has, so only // new ones are marked. With no lookup it does nothing. func markRepeats( ctx context.Context, logger *slog.Logger, lookup tenderStore, opportunities []*types.Opportunity, ) error { if lookup == nil || len(opportunities) == 0 { return nil } earliest := opportunities[0].LatestReleaseDate latest := earliest for _, opportunity := range opportunities { earliest = minTime(earliest, opportunity.LatestReleaseDate) latest = maxTime(latest, opportunity.LatestReleaseDate) } others, err := lookup.ListOtherSourcesTitles( ctx, euSupplySource, earliest.Add(-repeatWindow), latest.Add(repeatWindow+time.Nanosecond), ) if err != nil { return err } normalised := make([]string, len(others)) for i, other := range others { normalised[i] = normaliseTitle(other.Title) } repeats := 0 for _, opportunity := range opportunities { title := normaliseTitle(opportunity.Title) for i, other := range others { gap := other.LatestReleaseDate.Sub(opportunity.LatestReleaseDate) if gap.Abs() > repeatWindow || !similarTitles(title, normalised[i]) { continue } repeated, err := lookup.GetOpportunity(ctx, other.UUID) if err != nil { return fmt.Errorf("reading opportunity %s: %w", other.UUID, err) } if !confirmedRepeat(opportunity, repeated) { tagged(logger).Debug( "title matched, but not buyer or description", "uuid", opportunity.UUID, "title_matches", other.UUID, ) continue } opportunity.Marks.Hidden = true opportunity.Marks.Skipped = true repeats++ tagged(logger).Debug( "hid repeated opportunity", "uuid", opportunity.UUID, "repeats", other.UUID, ) break } } tagged(logger).Info( "hid repeated opportunities", "source", euSupplySource, "count", repeats, ) return nil } // minTime returns the earlier of two times. func minTime(first, second time.Time) time.Time { if second.Before(first) { return second } return first } // maxTime returns the later of two times. func maxTime(first, second time.Time) time.Time { if second.After(first) { return second } return first }