job-search/internal/eusupply/eusupply.go

// Package eusupply scrapes the public tenders listed on EU Supply's UK
// portal (uk.eu-supply.com), and converts them to opportunities.
//
// The site has no API: the client posts the portal's search form and reads
// the HTML it returns, then reads each tender's own page for its
// description and CPV codes. The client limits its own request rate (see
// WithRequestGap), as the site publishes no limit to wait out.
package eusupply

import (
	"cmp"
	"context"
	"errors"
	"fmt"
	"io"
	"log/slog"
	"math/rand/v2"
	"net/http"
	"net/http/cookiejar"
	"net/url"
	"strings"
	"sync"
	"time"
	_ "time/tzdata" // the portal's times are UK time, wherever this runs
)

// The defaults of the client's options.
const (
	// DefaultBaseURL is the UK portal's root.
	DefaultBaseURL = "https://uk.eu-supply.com"
	// DefaultMinRequestGap and DefaultMaxRequestGap bound the random wait
	// between two requests (see WithRequestGap).
	DefaultMinRequestGap = 2 * time.Second
	DefaultMaxRequestGap = 10 * time.Second
	// requestTimeout is how long one request may take.
	requestTimeout = 60 * time.Second
)

// Source is the Source of the opportunities Convert makes.
const Source = "EU Supply - UK"

// ErrUnexpectedStatus wraps a response status other than 200 OK.
var ErrUnexpectedStatus = errors.New("eusupply: unexpected response status")

// ukTime is the zone of the portal's dates and times.
var ukTime = mustLoadLocation("Europe/London")

// Client scrapes one EU Supply portal. Create it with New; it is safe for
// concurrent use, though its requests are made one at a time.
type Client struct {
	baseURL    string
	httpClient *http.Client
	log        *slog.Logger

	minGap, maxGap time.Duration

	// mu serialises requests, so the gap between them holds; lastRequest is
	// when the previous one finished (zero: none yet).
	mu          sync.Mutex
	lastRequest time.Time
}

// Option configures a Client.
type Option func(*Client)

// WithLogger makes the client log to logger instead of slog.Default().
func WithLogger(logger *slog.Logger) Option {
	return func(c *Client) {
		c.log = logger
	}
}

// WithRequestGap makes the client wait a random time from minGap to maxGap
// after each request before making the next. A maxGap less than minGap is
// read as minGap; 0 or less for both means no wait, for tests.
func WithRequestGap(minGap, maxGap time.Duration) Option {
	return func(c *Client) {
		c.minGap = minGap
		c.maxGap = max(minGap, maxGap)
	}
}

// New returns a client for the portal at baseURL (DefaultBaseURL if empty).
// It waits DefaultMinRequestGap to DefaultMaxRequestGap between requests
// unless given WithRequestGap, and logs to slog.Default() unless given
// WithLogger.
func New(baseURL string, opts ...Option) *Client {
	jar, _ := cookiejar.New(nil) // cookiejar.New never returns an error
	client := &Client{
		baseURL: strings.TrimSuffix(cmp.Or(baseURL, DefaultBaseURL), "/"),
		httpClient: &http.Client{
			Timeout: requestTimeout,
			Jar:     jar,
		},
		minGap: DefaultMinRequestGap,
		maxGap: DefaultMaxRequestGap,
	}
	for _, opt := range opts {
		opt(client)
	}

	return client
}

// logger returns the logger the client logs to, tagged with its component.
func (c *Client) logger() *slog.Logger {
	log := c.log
	if log == nil {
		log = slog.Default()
	}
	return log.With("component", "eusupply")
}

// do makes one request, once the gap since the previous request has passed,
// and returns its body. A status other than 200 OK fails with
// ErrUnexpectedStatus. It returns the request context's error if that is
// cancelled while it waits.
func (c *Client) do(req *http.Request) ([]byte, error) {
	c.mu.Lock()
	defer c.mu.Unlock()

	if err := c.wait(req.Context()); err != nil {
		return nil, err
	}
	defer func() {
		c.lastRequest = time.Now()
	}()

	c.logger().Info("fetching", "method", req.Method, "uri", req.URL.String())

	res, err := c.httpClient.Do(req)
	if err != nil {
		return nil, err
	}
	defer res.Body.Close() //nolint:errcheck // read only

	if res.StatusCode != http.StatusOK {
		return nil, fmt.Errorf("%w: %s", ErrUnexpectedStatus, res.Status)
	}

	body, err := io.ReadAll(res.Body)
	if err != nil {
		return nil, fmt.Errorf("reading %s: %w", req.URL, err)
	}

	return body, nil
}

// wait sleeps until a random gap (see WithRequestGap) has passed since the
// previous request, or ctx is cancelled. c.mu must be held.
func (c *Client) wait(ctx context.Context) error {
	if c.lastRequest.IsZero() || c.maxGap <= 0 {
		return ctx.Err()
	}

	gap := c.minGap
	if c.maxGap > c.minGap {
		//nolint:gosec // G404: a jitter, not a secret
		gap += rand.N(c.maxGap - c.minGap + 1)
	}

	timer := time.NewTimer(time.Until(c.lastRequest.Add(gap)))
	defer timer.Stop()

	select {
	case <-timer.C:
		return nil
	case <-ctx.Done():
		return ctx.Err()
	}
}

// get makes a GET request for the portal's path.
func (c *Client) get(ctx context.Context, path string) ([]byte, error) {
	req, err := http.NewRequestWithContext(
		ctx, http.MethodGet, c.baseURL+path, nil,
	)
	if err != nil {
		return nil, err
	}
	return c.do(req)
}

// post posts form to the portal's path.
func (c *Client) post(
	ctx context.Context, path string, form url.Values,
) ([]byte, error) {
	req, err := http.NewRequestWithContext(
		ctx,
		http.MethodPost,
		c.baseURL+path,
		strings.NewReader(form.Encode()),
	)
	if err != nil {
		return nil, err
	}
	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
	return c.do(req)
}

// mustLoadLocation loads a time zone known to be in the embedded tzdata.
func mustLoadLocation(name string) *time.Location {
	location, err := time.LoadLocation(name)
	if err != nil {
		panic(err)
	}
	return location
}