bwilliams
5cb3601e1f
Fetching: - Add `fetch service`, which runs a worker per enabled source until SIGINT/SIGTERM (internal/fetcher) - Add util.Worker: runs a task on an interval with a since-checkpoint that only advances on success, and a shutdown timeout for in-flight work; plus AwaitTask/AwaitTaskE for calls that ignore cancellation - Add fts_mock and sleeper sources - Sleeper: sleeps periodically to emulate work being done. Can be instructed to block the shutdown signal or handle it cleanly Mock FTS server (cmd/mock-fts, internal/mock/findATender): - Serves GET /api/1.0/ocdsReleasePackages with FTS paging, query validation, UK local time handling and per-client rate limiting - Generates fictional releases from an embedded catalog, moving procedures through planning, tender and award, and keeps them in a JSON Lines store that survives restarts - Add logging.AccessLog for combined/common access logs
101 lines
2.2 KiB
Go
101 lines
2.2 KiB
Go
package logging
|
|
|
|
import (
|
|
"fmt"
|
|
"log"
|
|
"net"
|
|
"net/http"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
)
|
|
|
|
// ---------- Access logging ----------
|
|
|
|
// statusRecorder captures the status code and body size for the access log.
|
|
type statusRecorder struct {
|
|
http.ResponseWriter
|
|
status int
|
|
bytes int
|
|
}
|
|
|
|
func (r *statusRecorder) WriteHeader(code int) {
|
|
if r.status == 0 {
|
|
r.status = code
|
|
}
|
|
r.ResponseWriter.WriteHeader(code)
|
|
}
|
|
|
|
func (r *statusRecorder) Write(b []byte) (int, error) {
|
|
if r.status == 0 {
|
|
r.status = http.StatusOK
|
|
}
|
|
n, err := r.ResponseWriter.Write(b)
|
|
r.bytes += n
|
|
return n, err
|
|
}
|
|
|
|
// logField escapes a value for a quoted log field the way nginx does
|
|
// (quotes, backslashes and control characters become \xHH) and uses "-"
|
|
// for empty values.
|
|
func logField(s string) string {
|
|
if s == "" {
|
|
return "-"
|
|
}
|
|
var b strings.Builder
|
|
for i := 0; i < len(s); i++ {
|
|
c := s[i]
|
|
if c == '"' || c == '\\' || c < 0x20 || c == 0x7f {
|
|
fmt.Fprintf(&b, "\\x%02X", c)
|
|
} else {
|
|
b.WriteByte(c)
|
|
}
|
|
}
|
|
return b.String()
|
|
}
|
|
|
|
// clientKey attempts to produce a client host name as a key identifier to split
|
|
// requests in the log between sources. Could be extended to also check
|
|
// Forwarded-For or other proxy headers
|
|
func clientKey(req *http.Request) string {
|
|
host, _, err := net.SplitHostPort(req.RemoteAddr)
|
|
if err != nil {
|
|
return req.RemoteAddr
|
|
}
|
|
return host
|
|
}
|
|
|
|
func AccessLog(logger *log.Logger, combined bool, next http.Handler) http.Handler {
|
|
return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
|
|
start := time.Now()
|
|
rec := &statusRecorder{ResponseWriter: w}
|
|
next.ServeHTTP(rec, req)
|
|
if rec.status == 0 {
|
|
rec.status = http.StatusOK
|
|
}
|
|
|
|
user := "-"
|
|
if u, _, ok := req.BasicAuth(); ok && u != "" {
|
|
user = logField(u)
|
|
}
|
|
size := "-" // CLF uses "-" for an empty body
|
|
if rec.bytes > 0 {
|
|
size = strconv.Itoa(rec.bytes)
|
|
}
|
|
line := fmt.Sprintf(`%s - %s [%s] "%s" %d %s`,
|
|
clientKey(req),
|
|
user,
|
|
start.Format("02/Jan/2006:15:04:05 -0700"),
|
|
logField(req.Method+" "+req.RequestURI+" "+req.Proto),
|
|
rec.status,
|
|
size,
|
|
)
|
|
if combined {
|
|
line += fmt.Sprintf(
|
|
` "%s" "%s"`, logField(req.Referer()), logField(req.UserAgent()),
|
|
)
|
|
}
|
|
logger.Println(line)
|
|
})
|
|
}
|