Added fetch service and a mock Find a Tender server
Fetching: - Add `fetch service`, which runs a worker per enabled source until SIGINT/SIGTERM (internal/fetcher) - Add util.Worker: runs a task on an interval with a since-checkpoint that only advances on success, and a shutdown timeout for in-flight work; plus AwaitTask/AwaitTaskE for calls that ignore cancellation - Add fts_mock and sleeper sources - Sleeper: sleeps periodically to emulate work being done. Can be instructed to block the shutdown signal or handle it cleanly Mock FTS server (cmd/mock-fts, internal/mock/findATender): - Serves GET /api/1.0/ocdsReleasePackages with FTS paging, query validation, UK local time handling and per-client rate limiting - Generates fictional releases from an embedded catalog, moving procedures through planning, tender and award, and keeps them in a JSON Lines store that survives restarts - Add logging.AccessLog for combined/common access logs
This commit is contained in:
1 parent
dcb714dc19
commit
5cb3601e1f
23 files changed
+97717
-3
No files matched your search
@@ -0,0 +1,100 @@
|
||||
package logging
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"log"
|
||||
"net"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// ---------- Access logging ----------
|
||||
|
||||
// statusRecorder captures the status code and body size for the access log.
|
||||
type statusRecorder struct {
|
||||
http.ResponseWriter
|
||||
status int
|
||||
bytes int
|
||||
}
|
||||
|
||||
func (r *statusRecorder) WriteHeader(code int) {
|
||||
if r.status == 0 {
|
||||
r.status = code
|
||||
}
|
||||
r.ResponseWriter.WriteHeader(code)
|
||||
}
|
||||
|
||||
func (r *statusRecorder) Write(b []byte) (int, error) {
|
||||
if r.status == 0 {
|
||||
r.status = http.StatusOK
|
||||
}
|
||||
n, err := r.ResponseWriter.Write(b)
|
||||
r.bytes += n
|
||||
return n, err
|
||||
}
|
||||
|
||||
// logField escapes a value for a quoted log field the way nginx does
|
||||
// (quotes, backslashes and control characters become \xHH) and uses "-"
|
||||
// for empty values.
|
||||
func logField(s string) string {
|
||||
if s == "" {
|
||||
return "-"
|
||||
}
|
||||
var b strings.Builder
|
||||
for i := 0; i < len(s); i++ {
|
||||
c := s[i]
|
||||
if c == '"' || c == '\\' || c < 0x20 || c == 0x7f {
|
||||
fmt.Fprintf(&b, "\\x%02X", c)
|
||||
} else {
|
||||
b.WriteByte(c)
|
||||
}
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// clientKey attempts to produce a client host name as a key identifier to split
|
||||
// requests in the log between sources. Could be extended to also check
|
||||
// Forwarded-For or other proxy headers
|
||||
func clientKey(req *http.Request) string {
|
||||
host, _, err := net.SplitHostPort(req.RemoteAddr)
|
||||
if err != nil {
|
||||
return req.RemoteAddr
|
||||
}
|
||||
return host
|
||||
}
|
||||
|
||||
func AccessLog(logger *log.Logger, combined bool, next http.Handler) http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
|
||||
start := time.Now()
|
||||
rec := &statusRecorder{ResponseWriter: w}
|
||||
next.ServeHTTP(rec, req)
|
||||
if rec.status == 0 {
|
||||
rec.status = http.StatusOK
|
||||
}
|
||||
|
||||
user := "-"
|
||||
if u, _, ok := req.BasicAuth(); ok && u != "" {
|
||||
user = logField(u)
|
||||
}
|
||||
size := "-" // CLF uses "-" for an empty body
|
||||
if rec.bytes > 0 {
|
||||
size = strconv.Itoa(rec.bytes)
|
||||
}
|
||||
line := fmt.Sprintf(`%s - %s [%s] "%s" %d %s`,
|
||||
clientKey(req),
|
||||
user,
|
||||
start.Format("02/Jan/2006:15:04:05 -0700"),
|
||||
logField(req.Method+" "+req.RequestURI+" "+req.Proto),
|
||||
rec.status,
|
||||
size,
|
||||
)
|
||||
if combined {
|
||||
line += fmt.Sprintf(
|
||||
` "%s" "%s"`, logField(req.Referer()), logField(req.UserAgent()),
|
||||
)
|
||||
}
|
||||
logger.Println(line)
|
||||
})
|
||||
}
|
||||
Reference in new issue
Block a user