package eusupply import ( "bytes" "context" "errors" "fmt" "net/url" "regexp" "strconv" "strings" "time" "golang.org/x/net/html" ) // The search form's endpoint and paging. const ( // searchPath is where the portal's public search form posts to. searchPath = "/ctm/Supplier/PublicTenders" // pageSize is the most results per page the form offers. pageSize = 100 // defaultSearchDays is how far back a search with no from date goes, // as the old scraper's first search did. defaultSearchDays = 7 // dateLayout is the form's and the results' date layout; dateTimeLayout // adds the time of day the results give in their cells' titles. dateLayout = "02/01/2006" dateTimeLayout = "02/01/2006 15:04:05" ) // ErrNoResults means a search page had no results table, which the portal // always returns, even for no tenders: the page has changed. var ErrNoResults = errors.New("eusupply: no search results table on the page") // pidPattern finds a tender's portal id (PID) in its link. var pidPattern = regexp.MustCompile(`[?&]PID=(\d+)`) // pagesPattern reads the number of pages from the pager's "of N". var pagesPattern = regexp.MustCompile(`^of (\d+)$`) // Tender is one public tender on the portal. The fields up to Countries are // from the search results; the rest are from the tender's own page (see // Client.Details), and are empty until it is read. type Tender struct { // ID is the quote/tender id the portal shows ("100114946"). ID string `json:"id"` // PID is the portal's internal id, which its pages are addressed by. PID string `json:"pid"` Reference string `json:"reference"` Title string `json:"title"` URL string `json:"url"` Published time.Time `json:"published"` Deadline *time.Time `json:"deadline"` Process string `json:"process"` Buyer string `json:"buyer"` Countries string `json:"countries"` // HasDetails is set once the tender's own page has been read. HasDetails bool `json:"has_details"` Description string `json:"description,omitempty"` ContractType string `json:"contract_type,omitempty"` CPVCodes []CPVCode `json:"cpv_codes,omitempty"` BuyerAddress []string `json:"buyer_address,omitempty"` Lots int `json:"lots,omitempty"` } // CPVCode is one of a tender's CPV codes, as its page lists them: the code // with its check digit ("48000000-8") and its description. type CPVCode struct { Code string `json:"code"` Description string `json:"description"` } // Search returns the tenders published from the date of from to the date // of until (UK dates, both included), reading every page of results. A // zero from searches the last 7 days; a zero until searches up to today. If a page // fails, the tenders of the pages before it are returned with the error. func (c *Client) Search( ctx context.Context, from, until time.Time, ) ([]Tender, error) { if until.IsZero() { until = time.Now() } if from.IsZero() { from = until.AddDate(0, 0, -defaultSearchDays) } var tenders []Tender for page := 1; ; page++ { body, err := c.post(ctx, searchPath, searchForm(from, until, page)) if err != nil { return tenders, fmt.Errorf("searching page %d: %w", page, err) } found, pages, err := c.parseSearchPage(body) if err != nil { return tenders, fmt.Errorf("reading search page %d: %w", page, err) } tenders = append(tenders, found...) if page >= pages { return tenders, nil } } } // searchForm returns the search form's fields for one page of the tenders // published from from to until. They are the fields the portal's own form // posts, with its defaults; the portal answers 500 if one is missing or // isn't valid. func searchForm(from, until time.Time, page int) url.Values { return url.Values{ "SearchFilter.SortField": {"None"}, "SearchFilter.SortDirection": {"None"}, "SearchFilter.ShortDescription": {""}, "SearchFilter.Reference": {""}, "SearchFilter.TenderId": {"0"}, "SearchFilter.OperatorId": {"0"}, "Branding": {"UK"}, "SavedCategoryId": {"0"}, "SavedUnitAndName": {""}, "SearchFilter.PublishType": {"1"}, // issued, not awarded "SearchFilter.BrandingCode": {""}, "TextFilter": {""}, "SearchFilter.FromDate": {from.In(ukTime).Format(dateLayout)}, "SearchFilter.ToDate": {until.In(ukTime).Format(dateLayout)}, "CpvContainer.CpvCodes": {""}, "CpvContainer.CpvMain": {""}, "CpvContainer.ContractType": {""}, "CpvContainer.CpvIds": {""}, "CpvContainer.IsMandatory": {"True"}, "SearchFilter.ShowExpiredRft": {"false"}, "SearchFilter.PagingInfo.PageNumber": {strconv.Itoa(page)}, "SearchFilter.PagingInfo.PageSize": {strconv.Itoa(pageSize)}, } } // parseSearchPage reads the tenders on a page of search results, and the // number of pages of results. A row that can't be read is logged and // skipped. func (c *Client) parseSearchPage(body []byte) ([]Tender, int, error) { document, err := html.Parse(bytes.NewReader(body)) if err != nil { return nil, 0, err } container := findFirst(document, hasID("searchResultContainer")) if container == nil { return nil, 0, ErrNoResults } table := findFirst(container, isTag("table")) if table == nil { return nil, 0, ErrNoResults } var tenders []Tender for _, body := range children(table, "tbody") { for _, row := range children(body, "tr") { tender, err := c.parseResultRow(row) if err != nil { c.logger().Warn( "skipping unreadable search result", "row", text(row), "error", err, ) continue } tenders = append(tenders, tender) } } return tenders, searchPages(document), nil } // parseResultRow reads one row of search results. Its cells are the id, // reference, title (a link to the tender), publication date, response // deadline, process, buyers and countries; each date cell's title has the // full time. func (c *Client) parseResultRow(row *html.Node) (Tender, error) { cells := children(row, "td") if len(cells) < 8 { return Tender{}, fmt.Errorf( "eusupply: a result row has %d cells, not 8", len(cells), ) } link := findFirst(cells[2], isTag("a")) if link == nil { return Tender{}, errors.New("eusupply: a result row has no link") } match := pidPattern.FindStringSubmatch(attr(link, "href")) if match == nil { return Tender{}, fmt.Errorf( "eusupply: no PID in the link %q", attr(link, "href"), ) } published, err := cellTime(cells[3]) if err != nil { return Tender{}, fmt.Errorf("publication date: %w", err) } tender := Tender{ ID: text(cells[0]), PID: match[1], Reference: text(cells[1]), Title: text(link), URL: c.tenderURL(match[1]), Published: published, Process: text(cells[5]), Buyer: text(cells[6]), Countries: text(cells[7]), } if tender.ID == "" { return Tender{}, errors.New("eusupply: a result row has no id") } // A tender may have no deadline yet (e.g. market engagement). if deadline, err := cellTime(cells[4]); err == nil { tender.Deadline = &deadline } return tender, nil } // tenderURL returns the address of the public page of the tender with pid. func (c *Client) tenderURL(pid string) string { return c.baseURL + tenderPath(pid) } // tenderPath returns the path of the public page of the tender with pid, // which the search results' links redirect to. func tenderPath(pid string) string { return "/ctm/Supplier/PublicPurchase/" + url.PathEscape(pid) + "/0/0?returnUrl=ctm/Supplier/PublicTenders&b=UK" } // cellTime reads the UK time in the title of a result cell's tooltip. func cellTime(cell *html.Node) (time.Time, error) { tooltip := findFirst(cell, hasClass("div", "js-tooltip-content")) if tooltip == nil { return time.Time{}, errors.New("eusupply: no time in the cell") } return parseUKTime(attr(tooltip, "title")) } // parseUKTime reads a portal time ("21/10/2026 12:45:00"; the seconds may // be left out) in UK time. func parseUKTime(value string) (time.Time, error) { value = strings.TrimSpace(value) parsed, err := time.ParseInLocation(dateTimeLayout, value, ukTime) if err != nil { parsed, err = time.ParseInLocation("02/01/2006 15:04", value, ukTime) } if err != nil { return time.Time{}, fmt.Errorf("eusupply: time %q: %w", value, err) } return parsed, nil } // searchPages reads the number of pages from the pager ("Page 1 of 3"), or // 1 if there is no pager. func searchPages(document *html.Node) int { for _, pager := range findAll(document, hasClass("ul", "ctm-pager-info")) { for _, item := range children(pager, "li") { match := pagesPattern.FindStringSubmatch(text(item)) if match == nil { continue } if pages, err := strconv.Atoi(match[1]); err == nil && pages > 0 { return pages } } } return 1 }