// Package eusupply scrapes the public tenders listed on EU Supply's UK // portal (uk.eu-supply.com), and converts them to opportunities. // // The site has no API: the client posts the portal's search form and reads // the HTML it returns, then reads each tender's own page for its // description and CPV codes. The client limits its own request rate (see // WithRequestGap), as the site publishes no limit to wait out. package eusupply import ( "cmp" "context" "errors" "fmt" "io" "log/slog" "math/rand/v2" "net/http" "net/http/cookiejar" "net/url" "strings" "sync" "time" _ "time/tzdata" // the portal's times are UK time, wherever this runs ) // The defaults of the client's options. const ( // DefaultBaseURL is the UK portal's root. DefaultBaseURL = "https://uk.eu-supply.com" // DefaultMinRequestGap and DefaultMaxRequestGap bound the random wait // between two requests (see WithRequestGap). DefaultMinRequestGap = 2 * time.Second DefaultMaxRequestGap = 10 * time.Second // requestTimeout is how long one request may take. requestTimeout = 60 * time.Second ) // Source is the Source of the opportunities Convert makes. const Source = "EU Supply - UK" // ErrUnexpectedStatus wraps a response status other than 200 OK. var ErrUnexpectedStatus = errors.New("eusupply: unexpected response status") // ukTime is the zone of the portal's dates and times. var ukTime = mustLoadLocation("Europe/London") // Client scrapes one EU Supply portal. Create it with New; it is safe for // concurrent use, though its requests are made one at a time. type Client struct { baseURL string httpClient *http.Client log *slog.Logger minGap, maxGap time.Duration // mu serialises requests, so the gap between them holds; lastRequest is // when the previous one finished (zero: none yet). mu sync.Mutex lastRequest time.Time } // Option configures a Client. type Option func(*Client) // WithLogger makes the client log to logger instead of slog.Default(). func WithLogger(logger *slog.Logger) Option { return func(c *Client) { c.log = logger } } // WithRequestGap makes the client wait a random time from minGap to maxGap // after each request before making the next. A maxGap less than minGap is // read as minGap; 0 or less for both means no wait, for tests. func WithRequestGap(minGap, maxGap time.Duration) Option { return func(c *Client) { c.minGap = minGap c.maxGap = max(minGap, maxGap) } } // New returns a client for the portal at baseURL (DefaultBaseURL if empty). // It waits DefaultMinRequestGap to DefaultMaxRequestGap between requests // unless given WithRequestGap, and logs to slog.Default() unless given // WithLogger. func New(baseURL string, opts ...Option) *Client { jar, _ := cookiejar.New(nil) // cookiejar.New never returns an error client := &Client{ baseURL: strings.TrimSuffix(cmp.Or(baseURL, DefaultBaseURL), "/"), httpClient: &http.Client{ Timeout: requestTimeout, Jar: jar, }, minGap: DefaultMinRequestGap, maxGap: DefaultMaxRequestGap, } for _, opt := range opts { opt(client) } return client } // logger returns the logger the client logs to, tagged with its component. func (c *Client) logger() *slog.Logger { log := c.log if log == nil { log = slog.Default() } return log.With("component", "eusupply") } // do makes one request, once the gap since the previous request has passed, // and returns its body. A status other than 200 OK fails with // ErrUnexpectedStatus. It returns the request context's error if that is // cancelled while it waits. func (c *Client) do(req *http.Request) ([]byte, error) { c.mu.Lock() defer c.mu.Unlock() if err := c.wait(req.Context()); err != nil { return nil, err } defer func() { c.lastRequest = time.Now() }() c.logger().Info("fetching", "method", req.Method, "uri", req.URL.String()) res, err := c.httpClient.Do(req) if err != nil { return nil, err } defer res.Body.Close() //nolint:errcheck // read only if res.StatusCode != http.StatusOK { return nil, fmt.Errorf("%w: %s", ErrUnexpectedStatus, res.Status) } body, err := io.ReadAll(res.Body) if err != nil { return nil, fmt.Errorf("reading %s: %w", req.URL, err) } return body, nil } // wait sleeps until a random gap (see WithRequestGap) has passed since the // previous request, or ctx is cancelled. c.mu must be held. func (c *Client) wait(ctx context.Context) error { if c.lastRequest.IsZero() || c.maxGap <= 0 { return ctx.Err() } gap := c.minGap if c.maxGap > c.minGap { //nolint:gosec // G404: a jitter, not a secret gap += rand.N(c.maxGap - c.minGap + 1) } timer := time.NewTimer(time.Until(c.lastRequest.Add(gap))) defer timer.Stop() select { case <-timer.C: return nil case <-ctx.Done(): return ctx.Err() } } // get makes a GET request for the portal's path. func (c *Client) get(ctx context.Context, path string) ([]byte, error) { req, err := http.NewRequestWithContext( ctx, http.MethodGet, c.baseURL+path, nil, ) if err != nil { return nil, err } return c.do(req) } // post posts form to the portal's path. func (c *Client) post( ctx context.Context, path string, form url.Values, ) ([]byte, error) { req, err := http.NewRequestWithContext( ctx, http.MethodPost, c.baseURL+path, strings.NewReader(form.Encode()), ) if err != nil { return nil, err } req.Header.Set("Content-Type", "application/x-www-form-urlencoded") return c.do(req) } // mustLoadLocation loads a time zone known to be in the embedded tzdata. func mustLoadLocation(name string) *time.Location { location, err := time.LoadLocation(name) if err != nil { panic(err) } return location }