package eusupply import ( "slices" "strings" "golang.org/x/net/html" ) // The helpers below read the parsed pages. The site's HTML is server // rendered, so the elements are found by their ids and classes, as the old // Node scraper found them with CSS selectors. // findAll returns the elements under node (node included) that match, in // document order. func findAll(node *html.Node, match func(*html.Node) bool) []*html.Node { var found []*html.Node var walk func(*html.Node) walk = func(current *html.Node) { if current.Type == html.ElementNode && match(current) { found = append(found, current) } for child := current.FirstChild; child != nil; child = child.NextSibling { walk(child) } } walk(node) return found } // findFirst returns the first element under node that matches, or nil. func findFirst(node *html.Node, match func(*html.Node) bool) *html.Node { found := findAll(node, match) if len(found) == 0 { return nil } return found[0] } // isTag matches elements with the tag name. func isTag(tag string) func(*html.Node) bool { return func(node *html.Node) bool { return node.Data == tag } } // hasID matches the element with the id. func hasID(id string) func(*html.Node) bool { return func(node *html.Node) bool { return attr(node, "id") == id } } // hasClass matches elements with tag (any, if empty) and the class among // their classes. func hasClass(tag, class string) func(*html.Node) bool { return func(node *html.Node) bool { if tag != "" && node.Data != tag { return false } return slices.Contains(strings.Fields(attr(node, "class")), class) } } // children returns node's element children with the tag name. func children(node *html.Node, tag string) []*html.Node { var found []*html.Node for child := node.FirstChild; child != nil; child = child.NextSibling { if child.Type == html.ElementNode && child.Data == tag { found = append(found, child) } } return found } // nextElement returns the element after node among its siblings, or nil. func nextElement(node *html.Node) *html.Node { sibling := node.NextSibling for sibling != nil && sibling.Type != html.ElementNode { sibling = sibling.NextSibling } return sibling } // attr returns the value of node's attribute key, or "". func attr(node *html.Node, key string) string { for _, attribute := range node.Attr { if attribute.Key == key { return attribute.Val } } return "" } // rawText returns the text under node as written, with a newline for each //
. func rawText(node *html.Node) string { var text strings.Builder var walk func(*html.Node) walk = func(current *html.Node) { switch { case current.Type == html.TextNode: text.WriteString(current.Data) case current.Type == html.ElementNode && current.Data == "br": text.WriteString("\n") } for child := current.FirstChild; child != nil; child = child.NextSibling { walk(child) } } walk(node) return text.String() } // text returns the text under node with its white space collapsed: runs of // spaces, non-breaking spaces and line breaks become one space, and the ends // are trimmed. func text(node *html.Node) string { return collapseSpace(rawText(node)) } // lines returns the non-empty lines of the text under node, each with its // white space collapsed. func lines(node *html.Node) []string { var found []string for line := range strings.SplitSeq(rawText(node), "\n") { if collapsed := collapseSpace(line); collapsed != "" { found = append(found, collapsed) } } return found } // collapseSpace collapses runs of white space (non-breaking spaces and zero // width spaces included) into one space, and trims the ends. func collapseSpace(value string) string { value = strings.ReplaceAll(value, "\u200b", "") return strings.Join(strings.Fields(value), " ") }