job-search/internal/eusupply/html.go

package eusupply

import (
	"slices"
	"strings"

	"golang.org/x/net/html"
)

// The helpers below read the parsed pages. The site's HTML is server
// rendered, so the elements are found by their ids and classes, as the old
// Node scraper found them with CSS selectors.

// findAll returns the elements under node (node included) that match, in
// document order.
func findAll(node *html.Node, match func(*html.Node) bool) []*html.Node {
	var found []*html.Node
	var walk func(*html.Node)
	walk = func(current *html.Node) {
		if current.Type == html.ElementNode && match(current) {
			found = append(found, current)
		}
		for child := current.FirstChild; child != nil; child = child.NextSibling {
			walk(child)
		}
	}
	walk(node)
	return found
}

// findFirst returns the first element under node that matches, or nil.
func findFirst(node *html.Node, match func(*html.Node) bool) *html.Node {
	found := findAll(node, match)
	if len(found) == 0 {
		return nil
	}
	return found[0]
}

// isTag matches elements with the tag name.
func isTag(tag string) func(*html.Node) bool {
	return func(node *html.Node) bool {
		return node.Data == tag
	}
}

// hasID matches the element with the id.
func hasID(id string) func(*html.Node) bool {
	return func(node *html.Node) bool {
		return attr(node, "id") == id
	}
}

// hasClass matches elements with tag (any, if empty) and the class among
// their classes.
func hasClass(tag, class string) func(*html.Node) bool {
	return func(node *html.Node) bool {
		if tag != "" && node.Data != tag {
			return false
		}
		return slices.Contains(strings.Fields(attr(node, "class")), class)
	}
}

// children returns node's element children with the tag name.
func children(node *html.Node, tag string) []*html.Node {
	var found []*html.Node
	for child := node.FirstChild; child != nil; child = child.NextSibling {
		if child.Type == html.ElementNode && child.Data == tag {
			found = append(found, child)
		}
	}
	return found
}

// nextElement returns the element after node among its siblings, or nil.
func nextElement(node *html.Node) *html.Node {
	sibling := node.NextSibling
	for sibling != nil && sibling.Type != html.ElementNode {
		sibling = sibling.NextSibling
	}
	return sibling
}

// attr returns the value of node's attribute key, or "".
func attr(node *html.Node, key string) string {
	for _, attribute := range node.Attr {
		if attribute.Key == key {
			return attribute.Val
		}
	}
	return ""
}

// rawText returns the text under node as written, with a newline for each
// <br>.
func rawText(node *html.Node) string {
	var text strings.Builder
	var walk func(*html.Node)
	walk = func(current *html.Node) {
		switch {
		case current.Type == html.TextNode:
			text.WriteString(current.Data)
		case current.Type == html.ElementNode && current.Data == "br":
			text.WriteString("\n")
		}
		for child := current.FirstChild; child != nil; child = child.NextSibling {
			walk(child)
		}
	}
	walk(node)
	return text.String()
}

// text returns the text under node with its white space collapsed: runs of
// spaces, non-breaking spaces and line breaks become one space, and the ends
// are trimmed.
func text(node *html.Node) string {
	return collapseSpace(rawText(node))
}

// lines returns the non-empty lines of the text under node, each with its
// white space collapsed.
func lines(node *html.Node) []string {
	var found []string
	for line := range strings.SplitSeq(rawText(node), "\n") {
		if collapsed := collapseSpace(line); collapsed != "" {
			found = append(found, collapsed)
		}
	}
	return found
}

// collapseSpace collapses runs of white space (non-breaking spaces and zero
// width spaces included) into one space, and trims the ends.
func collapseSpace(value string) string {
	value = strings.ReplaceAll(value, "\u200b", "")
	return strings.Join(strings.Fields(value), " ")
}