job-search/internal/fetcher/worker_eusupply.go

package fetcher

import (
	"context"
	"errors"
	"fmt"
	"log/slog"
	"time"

	"git.avsdev.uk/Go/job-search/internal/config"
	"git.avsdev.uk/Go/job-search/internal/eusupply"
	"git.avsdev.uk/Go/job-search/internal/store"
	"git.avsdev.uk/Go/job-search/internal/types"
	"git.avsdev.uk/Go/job-search/internal/util"
)

// euSupplySource is the Source of the opportunities the EU Supply source
// fetches.
const euSupplySource = eusupply.Source

// repeatWindow is how far apart an EU Supply tender's publication and
// another source's opportunity's latest release may be for the two to be
// the same (see markRepeats).
const repeatWindow = 30 * 24 * time.Hour

// tenderStore is what the EU Supply source reads from the store, to skip
// tenders already stored and to find the ones another source has.
type tenderStore interface {
	ReleaseExists(ctx context.Context, uuid, releaseID string) (bool, error)
	ListOtherSourcesTitles(
		ctx context.Context, source string, from, before time.Time,
	) ([]store.OpportunityTitle, error)
	GetOpportunity(ctx context.Context, uuid string) (*types.Opportunity, error)
}

// runEUSupply scrapes EU Supply's UK portal every configured interval until
// ctx is cancelled, passing what each run fetches to save, even when the run
// fails partway. A long window is backfilled first (see backfill). One
// client is used throughout, so its gap between requests holds across runs.
// lookup is nil without a store: every tender is then new, and none is
// checked for repeats.
func runEUSupply(
	ctx context.Context,
	logger *slog.Logger,
	conf *config.JSCSourceEUSupply,
	lookup tenderStore,
	save saveFunc,
	since time.Time,
) error {
	client := eusupply.New(
		conf.BaseURL,
		eusupply.WithLogger(logger),
		eusupply.WithRequestGap(conf.RequestMinGap, conf.RequestMaxGap),
	)
	fetch := func(
		ctx context.Context, since, until time.Time,
	) ([]*types.Opportunity, error) {
		return fetchEUSupply(ctx, logger, client, lookup, since, until)
	}

	since = backfill(
		ctx,
		logger,
		euSupplySource,
		conf.Interval,
		fetch,
		save,
		since,
	)

	worker := util.Worker{
		Logger:   tagged(logger),
		Name:     euSupplySource,
		Interval: conf.Interval,
		Task: func(ctx context.Context, since time.Time) error {
			opportunities, err := fetch(ctx, since, time.Time{})
			return saveFetched(ctx, logger, save, opportunities, err)
		},
	}
	return worker.Run(ctx, since)
}

// fetchEUSupply fetches the tenders published from since up to until (zero:
// today) that aren't stored yet, with their own pages read, and marks the
// ones another source already has Hidden and Skipped (see markRepeats). If
// it fails partway, the tenders read in full before the error are returned
// along with it. Finding nothing new returns ErrNoNewReleases.
func fetchEUSupply(
	ctx context.Context,
	logger *slog.Logger,
	client *eusupply.Client,
	lookup tenderStore,
	since, until time.Time,
) ([]*types.Opportunity, error) {
	tenders, fetchErr := client.Search(ctx, since, until)

	fresh, err := newTenders(ctx, lookup, tenders)
	if err != nil {
		return nil, err
	}

	for i := range fresh {
		if err := client.Details(ctx, &fresh[i]); err != nil {
			fresh = fresh[:i]
			fetchErr = errors.Join(fetchErr, err)
			break
		}
	}

	opportunities := eusupply.Convert(fresh)
	if err := markRepeats(ctx, logger, lookup, opportunities); err != nil {
		// Saved unmarked, a repeat would stay visible, as marks are only
		// stored for a new opportunity: fetch them again next run instead.
		return nil, errors.Join(fetchErr, err)
	}

	tagged(logger).Info(
		"fetched opportunities",
		"source", euSupplySource,
		"listed", len(tenders),
		"count", len(opportunities),
	)

	if len(opportunities) == 0 && fetchErr == nil {
		return nil, ErrNoNewReleases
	}
	return opportunities, fetchErr
}

// newTenders returns the tenders whose release isn't stored yet, so only
// their pages are read. Saving a stored one again would replace its row
// with the search's fields alone. With no lookup, every tender is new.
func newTenders(
	ctx context.Context, lookup tenderStore, tenders []eusupply.Tender,
) ([]eusupply.Tender, error) {
	if lookup == nil {
		return tenders, nil
	}

	var fresh []eusupply.Tender
	for _, tender := range tenders {
		exists, err := lookup.ReleaseExists(
			ctx, eusupply.UUID(tender.ID), eusupply.ReleaseID(tender),
		)
		if err != nil {
			return nil, fmt.Errorf("checking tender %s: %w", tender.ID, err)
		}
		if !exists {
			fresh = append(fresh, tender)
		}
	}

	return fresh, nil
}

// markRepeats marks Hidden and Skipped each opportunity that repeats one
// from another source, most often its Find a Tender notice: its title is at
// least titleMatchThreshold alike (see similarTitles), its latest release
// is within repeatWindow, and its buyer or description matches too (see
// confirmedRepeat). Only the opportunities whose titles match are read in
// full. The store keeps the marks of an opportunity it already has, so only
// new ones are marked. With no lookup it does nothing.
func markRepeats(
	ctx context.Context,
	logger *slog.Logger,
	lookup tenderStore,
	opportunities []*types.Opportunity,
) error {
	if lookup == nil || len(opportunities) == 0 {
		return nil
	}

	earliest := opportunities[0].LatestReleaseDate
	latest := earliest
	for _, opportunity := range opportunities {
		earliest = minTime(earliest, opportunity.LatestReleaseDate)
		latest = maxTime(latest, opportunity.LatestReleaseDate)
	}
	others, err := lookup.ListOtherSourcesTitles(
		ctx,
		euSupplySource,
		earliest.Add(-repeatWindow),
		latest.Add(repeatWindow+time.Nanosecond),
	)
	if err != nil {
		return err
	}

	normalised := make([]string, len(others))
	for i, other := range others {
		normalised[i] = normaliseTitle(other.Title)
	}

	repeats := 0
	for _, opportunity := range opportunities {
		title := normaliseTitle(opportunity.Title)
		for i, other := range others {
			gap := other.LatestReleaseDate.Sub(opportunity.LatestReleaseDate)
			if gap.Abs() > repeatWindow || !similarTitles(title, normalised[i]) {
				continue
			}
			repeated, err := lookup.GetOpportunity(ctx, other.UUID)
			if err != nil {
				return fmt.Errorf("reading opportunity %s: %w", other.UUID, err)
			}
			if !confirmedRepeat(opportunity, repeated) {
				tagged(logger).Debug(
					"title matched, but not buyer or description",
					"uuid", opportunity.UUID,
					"title_matches", other.UUID,
				)
				continue
			}
			opportunity.Marks.Hidden = true
			opportunity.Marks.Skipped = true
			repeats++
			tagged(logger).Debug(
				"hid repeated opportunity",
				"uuid", opportunity.UUID,
				"repeats", other.UUID,
			)
			break
		}
	}

	tagged(logger).Info(
		"hid repeated opportunities",
		"source", euSupplySource,
		"count", repeats,
	)

	return nil
}

// minTime returns the earlier of two times.
func minTime(first, second time.Time) time.Time {
	if second.Before(first) {
		return second
	}
	return first
}

// maxTime returns the later of two times.
func maxTime(first, second time.Time) time.Time {
	if second.After(first) {
		return second
	}
	return first
}