diff --git a/README.md b/README.md index 26dfe81..19f1b39 100644 --- a/README.md +++ b/README.md @@ -15,8 +15,9 @@ scripts/build.sh # build/job-search- scripts/build.sh --release # also package release/job-search--.tar.gz ``` -The build script runs `go generate ./...` and stamps the version from the -latest git tag. A build from untagged commits is marked as a development build +The build script runs `go generate ./...`, which needs `swag` installed (see +[Installation of dev tools](#installation-of-dev-tools)), and stamps the +version from the latest git tag. A build from untagged commits is marked as a development build and isn't packaged unless `--force` is also given. During development, `go run -tags sqlite_fts5 ./cmd/jobsearch` works as well. @@ -69,6 +70,11 @@ Global flags: `--config`/`-c` (config file), `--debug`/`-d` and | `job-search migrate status` | Show the database version and which migrations are applied or pending | | `job-search version` | Print version information | +`job-search serve` describes its REST API in OpenAPI 3.1 at +`/api/openapi.json`, and shows it in Swagger UI at `/api/docs`. The +description is generated from the handlers' annotations in `internal/api` by +`go generate`. + The `migrate` commands work on the database in `database.file`. The store refuses to open a database that isn't at this build's schema version, so run `job-search migrate` after installing a new version. @@ -234,11 +240,15 @@ go install github.com/spf13/cobra-cli@latest go install git.avsdev.uk/Go/go-ocds/cmd/ocdsgen@latest go install github.com/pressly/goose/v3/cmd/goose@latest go install github.com/golangci/golangci-lint/v2/cmd/golangci-lint@latest +go install github.com/swaggo/swag/v2/cmd/swag@v2.0.0-rc6 ``` `goose` creates new migration files (see `internal/store`); the migrations themselves are applied by `job-search migrate`. `golangci-lint` uses the -project's `.golangci.json`. +project's `.golangci.json`. `swag` generates the API's OpenAPI 3.1 spec from +the handlers' comments (`swag init --v3.1`). Its v2, the version with OpenAPI +3 support, has only had release candidates so far, so it is pinned to one +rather than `@latest`. Ensure that the GOBIN path ($HOME/go/bin) is added to the environment before running any of the tools. diff --git a/internal/api/.gitignore b/internal/api/.gitignore new file mode 100644 index 0000000..52a3c6b --- /dev/null +++ b/internal/api/.gitignore @@ -0,0 +1 @@ +openapi/ diff --git a/internal/api/api.go b/internal/api/api.go new file mode 100644 index 0000000..d5b4eeb --- /dev/null +++ b/internal/api/api.go @@ -0,0 +1,45 @@ +package api + +import ( + "github.com/go-chi/chi/v5" + + "git.avsdev.uk/Go/job-search/internal/store" +) + +// API is the REST API, mounted under /api. Create it with New, giving it what +// its resources need as options, and register its routes with Routes. The +// server applies the API's middleware (CSRF, NoStore, RequireJSON, rate +// limits); the API only registers handlers. +type API struct { + store *store.Store // nil: no opportunity routes +} + +// Option configures the API. +type Option func(*API) + +// WithStore makes the API serve the opportunities in opportunityStore under +// /opportunities. Without it there are no opportunity routes. +func WithStore(opportunityStore *store.Store) Option { + return func(a *API) { + a.store = opportunityStore + } +} + +// New returns the API. +func New(opts ...Option) *API { + api := &API{} + for _, option := range opts { + option(api) + } + + return api +} + +// Routes registers the API's routes on router (mounted under /api): its +// description (see openapi.go), and the opportunities if it has a store. +func (a *API) Routes(router chi.Router) { + docsRoutes(router) + if a.store != nil { + newOpportunityRoutes(a.store).Routes(router) + } +} diff --git a/internal/api/common.go b/internal/api/common.go new file mode 100644 index 0000000..01b87cf --- /dev/null +++ b/internal/api/common.go @@ -0,0 +1,23 @@ +package api + +import ( + "fmt" + "strconv" +) + +// intParam parses the query parameter text as an integer from low to high, +// or returns fallback if it is empty. Its error is meant for the client. +func intParam(text string, fallback, low, high int) (int, error) { + if text == "" { + return fallback, nil + } + + number, err := strconv.Atoi(text) + if err != nil || number < low || number > high { + return 0, fmt.Errorf( + "parameter must be an integer between %d and %d", low, high, + ) + } + + return number, nil +} diff --git a/internal/api/doc.go b/internal/api/doc.go new file mode 100644 index 0000000..5ed76df --- /dev/null +++ b/internal/api/doc.go @@ -0,0 +1,24 @@ +// Package api implements the REST API mounted under /api. New assembles it +// (api.go): the opportunities, read-only apart from their marks +// (opportunities.go), and the API's own OpenAPI 3.1 description and docs +// viewer (openapi.go). New resources are added in New and Routes, so the +// server only mounts the API. +// +// The description is generated from the swag annotations on this package's +// handlers by go generate (swag v2, see the README's dev tools), into +// openapi/swagger.json. The annotations below are the API's general +// information. +// +// @title JobSearch API +// @version 1.0 +// @description Procurement opportunities fetched by JobSearch, and the user's +// @description marks on them. Errors are RFC 9457 problem details. +// @servers.url /api +// +// @securityDefinitions.apikey CSRF +// @in header +// @name X-CSRF-Token +// @description From GET /api/csrf; send its cookie too. +package api + +//go:generate swag init --v3.1 --generalInfo doc.go --dir . --parseInternal --parseDependencyLevel 1 --outputTypes json --output openapi diff --git a/internal/api/openapi.go b/internal/api/openapi.go new file mode 100644 index 0000000..cc802df --- /dev/null +++ b/internal/api/openapi.go @@ -0,0 +1,54 @@ +package api + +import ( + _ "embed" + "net/http" + + "github.com/go-chi/chi/v5" +) + +// openAPISpec is the API's OpenAPI 3.1 description, generated by go generate +// (see doc.go). +// +//go:embed openapi/swagger.json +var openAPISpec []byte + +// docsPage shows openapi.json, relative to /api/docs, in Swagger UI, loaded +// from jsDelivr at a fixed version. A content security policy +// (server.headers.content_security_policy) must allow that. +const docsPage = ` + + + + +JobSearch API + + + +
+ + + + +` + +// docsRoutes registers the API's description on router: GET /openapi.json, +// the OpenAPI 3.1 description, and GET /docs, a viewer of it. +func docsRoutes(router chi.Router) { + router.Get("/openapi.json", serveOpenAPISpec) + router.Get("/docs", serveDocsPage) +} + +// serveOpenAPISpec serves GET /openapi.json. +func serveOpenAPISpec(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.Write(openAPISpec) //nolint:errcheck,gosec // the client may have gone +} + +// serveDocsPage serves GET /docs. +func serveDocsPage(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/html; charset=utf-8") + w.Write([]byte(docsPage)) //nolint:errcheck,gosec // the client may have gone +} diff --git a/internal/api/openapi_test.go b/internal/api/openapi_test.go new file mode 100644 index 0000000..d8e2203 --- /dev/null +++ b/internal/api/openapi_test.go @@ -0,0 +1,136 @@ +package api + +import ( + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/go-chi/chi/v5" + + "git.avsdev.uk/Go/job-search/internal/store" +) + +// The description is served as JSON, and the docs page points at it. +func TestDocsRoutes(t *testing.T) { + router := chi.NewRouter() + New().Routes(router) + server := httptest.NewServer(router) + t.Cleanup(server.Close) + + res, err := http.Get(server.URL + "/openapi.json") + if err != nil { + t.Fatal(err) + } + defer res.Body.Close() + var spec struct { + OpenAPI string `json:"openapi"` + } + if err := json.NewDecoder(res.Body).Decode(&spec); err != nil { + t.Fatal(err) + } + if res.Header.Get("Content-Type") != "application/json" || + spec.OpenAPI != "3.1.0" { + t.Errorf( + "openapi.json: %s, openapi %q; want JSON, 3.1.0", + res.Header.Get("Content-Type"), spec.OpenAPI, + ) + } + + res, err = http.Get(server.URL + "/docs") + if err != nil { + t.Fatal(err) + } + defer res.Body.Close() + page, _ := io.ReadAll(res.Body) + if !strings.Contains(string(page), `url: "openapi.json"`) { + t.Errorf("docs page doesn't load openapi.json:\n%s", page) + } +} + +// Every route the API serves, apart from its description's own, is in the +// description, and the description has no route that isn't served, so the +// annotations can't fall behind as resources are added. +func TestOpenAPICoversRoutes(t *testing.T) { + var spec struct { + Paths map[string]map[string]json.RawMessage `json:"paths"` + } + if err := json.Unmarshal(openAPISpec, &spec); err != nil { + t.Fatal(err) + } + documented := map[string]bool{} + for path, operations := range spec.Paths { + for method := range operations { + documented[strings.ToUpper(method)+" "+path] = true + } + } + + // A store that is never opened: registering routes doesn't use it. + router := chi.NewRouter() + New(WithStore(&store.Store{})).Routes(router) + description := map[string]bool{ + "GET /openapi.json": true, + "GET /docs": true, + } + served := map[string]bool{} + err := chi.Walk(router, func( + method, route string, _ http.Handler, _ ...func(http.Handler) http.Handler, + ) error { + if !description[method+" "+route] { + served[method+" "+route] = true + } + return nil + }) + if err != nil { + t.Fatal(err) + } + + for route := range served { + if !documented[route] { + t.Errorf("%s is served but not in the OpenAPI description", route) + } + } + for route := range documented { + if !served[route] { + t.Errorf("%s is in the OpenAPI description but not served", route) + } + } +} + +// The description's list of marks matches OpportunityMarks, so its Enums +// annotation can't fall behind the struct. +func TestOpenAPIMarkEnum(t *testing.T) { + var spec struct { + Paths map[string]map[string]struct { + Parameters []struct { + Name string `json:"name"` + Schema struct { + Enum []string `json:"enum"` + } `json:"schema"` + } `json:"parameters"` + } `json:"paths"` + } + if err := json.Unmarshal(openAPISpec, &spec); err != nil { + t.Fatal(err) + } + + want := strings.Join(markNames, ",") + checked := 0 + operations := spec.Paths["/opportunities/{uuid}/marks/{mark}"] + for method, operation := range operations { + for _, parameter := range operation.Parameters { + if parameter.Name != "mark" { + continue + } + checked++ + if got := strings.Join(parameter.Schema.Enum, ","); got != want { + t.Errorf("%s mark enum = %s; want %s", method, got, want) + } + } + } + if checked != 2 { + t.Errorf("checked the mark of %d routes; want 2 (PUT, DELETE)", checked) + } +} diff --git a/internal/api/opportunities.go b/internal/api/opportunities.go new file mode 100644 index 0000000..3b31cd8 --- /dev/null +++ b/internal/api/opportunities.go @@ -0,0 +1,267 @@ +package api + +import ( + "errors" + "fmt" + "net/http" + "reflect" + "strings" + "sync" + + "github.com/go-chi/chi/v5" + + "git.avsdev.uk/Go/job-search/internal/httpx" + "git.avsdev.uk/Go/job-search/internal/store" + "git.avsdev.uk/Go/job-search/internal/types" +) + +// markIndex maps the name of each mark that can be set and cleared, as named +// in URLs, to the index of its field in types.OpportunityMarks. markNames +// lists the names in field order. Both are read from the struct (see +// findMarks), so a mark added to it can be changed over the API. +var markIndex, markNames = findMarks() + +// OpportunityPage is a page of opportunities, as GET /opportunities returns +// it. HasMore reports that another page follows, at Offset+Limit. +type OpportunityPage struct { + Opportunities []types.Opportunity `json:"opportunities"` + Offset int `json:"offset"` + Limit int `json:"limit"` + HasMore bool `json:"has_more"` +} + +// opportunityRoutes wires HTTP handlers to the opportunity store. +// Opportunities are read-only; only their marks can be changed. +type opportunityRoutes struct { + store *store.Store + + // marksMu makes changing one mark (read the marks, change one, write + // them all back) atomic, so concurrent changes to different marks of an + // opportunity don't undo each other. + marksMu sync.Mutex +} + +// newOpportunityRoutes returns the opportunity handlers for +// opportunityStore. +func newOpportunityRoutes(opportunityStore *store.Store) *opportunityRoutes { + return &opportunityRoutes{ + store: opportunityStore, + } +} + +// Routes registers the opportunity routes on router (mounted under /api). +func (h *opportunityRoutes) Routes(router chi.Router) { + router.Get("/opportunities", h.list) + router.Get("/opportunities/{uuid}", h.get) + router.Put("/opportunities/{uuid}/marks/{mark}", h.setMark) + router.Delete("/opportunities/{uuid}/marks/{mark}", h.clearMark) +} + +// list serves GET /opportunities: a page of opportunities, newest release +// first, with their classifications and marks but not their releases. The +// page is set by the offset and limit (default 50, at most 100) query +// parameters. +// +// @Summary List opportunities +// @Description A page of opportunities, newest release first, with their +// @Description classifications and marks but not their releases. has_more +// @Description says whether another page follows, at offset+limit. +// @Tags opportunities +// @Produce json +// @Param offset query int false "Opportunities to skip" minimum(0) default(0) +// @Param limit query int false "Page size" minimum(1) maximum(100) default(50) +// @Success 200 {object} OpportunityPage +// @Failure 400 {object} httpx.Problem "Invalid offset or limit" +// @Failure 500 {object} httpx.Problem +// @Router /opportunities [get] +func (h *opportunityRoutes) list(w http.ResponseWriter, r *http.Request) { + query := r.URL.Query() + offset, offsetErr := intParam(query.Get("offset"), 0, 0, 1<<30) + limit, limitErr := intParam(query.Get("limit"), 50, 1, 100) + err := errors.Join(offsetErr, limitErr) + if err != nil { + httpx.WriteProblem(w, r, http.StatusBadRequest, err.Error()) + return + } + + // One more than the page, to tell whether another page follows. + opportunities, err := h.store.ListOpportunities( + r.Context(), + store.OpportunityFilter{ + Limit: limit + 1, + Offset: offset, + }, + ) + if err != nil { + httpx.WriteProblem(w, r, http.StatusInternalServerError, "") + return + } + + page := OpportunityPage{ + Opportunities: opportunities, + Offset: offset, + Limit: limit, + HasMore: len(opportunities) > limit, + } + if page.HasMore { + page.Opportunities = opportunities[:limit] + } + + httpx.WriteJSON(w, http.StatusOK, page) +} + +// get serves GET /opportunities/{uuid}: one opportunity with its +// classifications, marks and releases. +// +// @Summary Get an opportunity +// @Description One opportunity with its classifications, marks and all of its +// @Description releases. +// @Tags opportunities +// @Produce json +// @Param uuid path string true "The opportunity's UUID (its OCID)" +// @Success 200 {object} types.Opportunity +// @Failure 404 {object} httpx.Problem "No such opportunity" +// @Failure 500 {object} httpx.Problem +// @Router /opportunities/{uuid} [get] +func (h *opportunityRoutes) get(w http.ResponseWriter, r *http.Request) { + opportunity, err := h.store.GetOpportunity( + r.Context(), chi.URLParam(r, "uuid"), + ) + if err != nil { + h.fail(w, r, err) + return + } + + httpx.WriteJSON(w, http.StatusOK, opportunity) +} + +// setMark serves PUT /opportunities/{uuid}/marks/{mark}, setting the mark. +// +// @Summary Set a mark +// @Description Sets one of the opportunity's marks, keeping the others, and +// @Description returns all of them. Setting a mark that is already set +// @Description changes nothing. won and lost exclude each other. +// @Tags marks +// @Produce json +// @Security CSRF +// @Param uuid path string true "The opportunity's UUID (its OCID)" +// @Param mark path string true "The mark" Enums(hidden, skipped, interested, bid_submitted, won, lost) +// @Success 200 {object} types.OpportunityMarks +// @Failure 403 {object} httpx.Problem "No valid CSRF token" +// @Failure 404 {object} httpx.Problem "No such opportunity or mark" +// @Failure 409 {object} httpx.Problem "won and lost together" +// @Failure 500 {object} httpx.Problem +// @Router /opportunities/{uuid}/marks/{mark} [put] +func (h *opportunityRoutes) setMark(w http.ResponseWriter, r *http.Request) { + h.changeMark(w, r, true) +} + +// clearMark serves DELETE /opportunities/{uuid}/marks/{mark}, clearing the +// mark. +// +// @Summary Clear a mark +// @Description Clears one of the opportunity's marks, keeping the others, and +// @Description returns all of them. Clearing a mark that isn't set changes +// @Description nothing. +// @Tags marks +// @Produce json +// @Security CSRF +// @Param uuid path string true "The opportunity's UUID (its OCID)" +// @Param mark path string true "The mark" Enums(hidden, skipped, interested, bid_submitted, won, lost) +// @Success 200 {object} types.OpportunityMarks +// @Failure 403 {object} httpx.Problem "No valid CSRF token" +// @Failure 404 {object} httpx.Problem "No such opportunity or mark" +// @Failure 500 {object} httpx.Problem +// @Router /opportunities/{uuid}/marks/{mark} [delete] +func (h *opportunityRoutes) clearMark(w http.ResponseWriter, r *http.Request) { + h.changeMark(w, r, false) +} + +// changeMark sets the mark named in the URL to value, keeping the +// opportunity's other marks, and responds with all of its marks. Setting won +// while lost is set, or lost while won is, is refused with 409 Conflict. +func (h *opportunityRoutes) changeMark( + w http.ResponseWriter, r *http.Request, value bool, +) { + uuid, name := chi.URLParam(r, "uuid"), chi.URLParam(r, "mark") + + h.marksMu.Lock() + defer h.marksMu.Unlock() + + opportunity, err := h.store.GetOpportunity(r.Context(), uuid) + if err != nil { + h.fail(w, r, err) + return + } + + marks := opportunity.Marks + marks.UUID = uuid + if !setMark(&marks, name, value) { + httpx.WriteProblem( + w, + r, + http.StatusNotFound, + fmt.Sprintf("no mark %q: marks are %v", name, markNames), + ) + return + } + + err = h.store.SetMarks(r.Context(), marks) + if err != nil { + h.fail(w, r, err) + return + } + + httpx.WriteJSON(w, http.StatusOK, marks) +} + +// findMarks returns the marks of types.OpportunityMarks: every bool field, +// named by its JSON tag, mapped to the field's index, and the names in field +// order. +func findMarks() (map[string]int, []string) { + index := map[string]int{} + var names []string + marksType := reflect.TypeFor[types.OpportunityMarks]() + for i := range marksType.NumField() { + field := marksType.Field(i) + if field.Type.Kind() != reflect.Bool { + continue + } + name, _, _ := strings.Cut(field.Tag.Get("json"), ",") + index[name] = i + names = append(names, name) + } + + return index, names +} + +// setMark sets the mark of marks named name (see markIndex) to value, and +// reports whether name is a mark. +func setMark(marks *types.OpportunityMarks, name string, value bool) bool { + i, ok := markIndex[name] + if !ok { + return false + } + + reflect.ValueOf(marks).Elem().Field(i).SetBool(value) + return true +} + +// fail writes the problem response for a store error. +func (h *opportunityRoutes) fail( + w http.ResponseWriter, r *http.Request, err error, +) { + switch { + case errors.Is(err, store.ErrNotFound): + httpx.WriteProblem(w, r, http.StatusNotFound, "opportunity not found") + case errors.Is(err, types.ErrWonAndLost): + httpx.WriteProblem( + w, + r, + http.StatusConflict, + "an opportunity cannot be both won and lost", + ) + default: + httpx.WriteProblem(w, r, http.StatusInternalServerError, "") + } +} diff --git a/internal/api/opportunities_test.go b/internal/api/opportunities_test.go new file mode 100644 index 0000000..f519c95 --- /dev/null +++ b/internal/api/opportunities_test.go @@ -0,0 +1,304 @@ +package api + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "github.com/go-chi/chi/v5" + + "git.avsdev.uk/Go/job-search/internal/store" + "git.avsdev.uk/Go/job-search/internal/types" +) + +// opportunitiesServer serves the opportunity routes over a store holding +// count opportunities, ocds-1 the oldest, and returns its URL. +func opportunitiesServer(t *testing.T, count int) string { + t.Helper() + opportunityStore, err := store.Open( + context.Background(), + filepath.Join(t.TempDir(), "api.db"), + store.WithAutoMigrate(), + ) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + opportunityStore.Close() + }) + + for i := 1; i <= count; i++ { + opportunity := &types.Opportunity{ + UUID: fmt.Sprintf("ocds-%d", i), + UUIDType: "ocid", + Title: fmt.Sprintf("Opportunity %d", i), + Source: "FindATender", + Releases: []types.OpportunityRelease{ + { + ReleaseID: "r1", + ReleaseDate: time.Date(2026, 10, i, 12, 0, 0, 0, time.UTC), + JSON: "{}", + }, + }, + } + _, err := opportunityStore.SaveOpportunity( + context.Background(), opportunity, + ) + if err != nil { + t.Fatal(err) + } + } + + router := chi.NewRouter() + New(WithStore(opportunityStore)).Routes(router) + server := httptest.NewServer(router) + t.Cleanup(server.Close) + return server.URL +} + +// request sends a bodyless request and decodes a JSON reply into reply (if +// not nil), returning the status. +func request(t *testing.T, method, url string, reply any) int { + t.Helper() + req, err := http.NewRequest(method, url, nil) + if err != nil { + t.Fatal(err) + } + res, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + defer res.Body.Close() + if reply != nil && res.StatusCode < 300 { + if err := json.NewDecoder(res.Body).Decode(reply); err != nil { + t.Fatal(err) + } + } + return res.StatusCode +} + +// uuids returns the UUIDs of a page's opportunities. +func uuids(page OpportunityPage) []string { + var ids []string + for _, opportunity := range page.Opportunities { + ids = append(ids, opportunity.UUID) + } + return ids +} + +// Opportunities are listed newest first, a page at a time. +func TestListOpportunitiesPaging(t *testing.T) { + url := opportunitiesServer(t, 5) + all := []string{"ocds-5", "ocds-4", "ocds-3", "ocds-2", "ocds-1"} + + tests := []struct { + query string + want []string + hasMore bool + }{ + {"?limit=2", []string{"ocds-5", "ocds-4"}, true}, + {"?limit=2&offset=2", []string{"ocds-3", "ocds-2"}, true}, + {"?limit=2&offset=4", []string{"ocds-1"}, false}, + {"?limit=5", all, false}, + {"?offset=10", nil, false}, + {"", all, false}, + } + for _, tt := range tests { + var page OpportunityPage + status := request(t, "GET", url+"/opportunities"+tt.query, &page) + if status != 200 { + t.Fatalf("%s: status %d", tt.query, status) + } + got := strings.Join(uuids(page), ",") + want := strings.Join(tt.want, ",") + if got != want || page.HasMore != tt.hasMore { + t.Errorf( + "%s: got %s, has_more %v; want %s, %v", + tt.query, got, page.HasMore, want, tt.hasMore, + ) + } + } + + var page OpportunityPage + request(t, "GET", url+"/opportunities", &page) + if page.Limit != 50 || page.Offset != 0 { + t.Errorf( + "defaults: limit %d, offset %d; want 50, 0", page.Limit, page.Offset, + ) + } +} + +// An empty page is [] in JSON, not null. +func TestListOpportunitiesEmpty(t *testing.T) { + url := opportunitiesServer(t, 0) + res, err := http.Get(url + "/opportunities") + if err != nil { + t.Fatal(err) + } + defer res.Body.Close() + var raw map[string]json.RawMessage + json.NewDecoder(res.Body).Decode(&raw) + if string(raw["opportunities"]) != "[]" { + t.Errorf("opportunities = %s; want []", raw["opportunities"]) + } +} + +func TestListOpportunitiesBadPaging(t *testing.T) { + url := opportunitiesServer(t, 1) + queries := []string{"?limit=0", "?limit=101", "?limit=x", "?offset=-1"} + for _, query := range queries { + status := request(t, "GET", url+"/opportunities"+query, nil) + if status != 400 { + t.Errorf("%s: status %d; want 400", query, status) + } + } +} + +// One opportunity comes with its releases; a missing one is 404. +func TestGetOpportunity(t *testing.T) { + url := opportunitiesServer(t, 2) + + var opportunity types.Opportunity + status := request(t, "GET", url+"/opportunities/ocds-2", &opportunity) + if status != 200 { + t.Fatalf("status %d", status) + } + if opportunity.UUID != "ocds-2" || len(opportunity.Releases) != 1 { + t.Errorf("got %+v; want ocds-2 with its release", opportunity) + } + + status = request(t, "GET", url+"/opportunities/nope", nil) + if status != 404 { + t.Errorf("missing: status %d; want 404", status) + } +} + +// Opportunities are read-only: there is no way to write one. +func TestOpportunitiesReadOnly(t *testing.T) { + url := opportunitiesServer(t, 1) + for _, method := range []string{"POST", "PUT", "PATCH", "DELETE"} { + status := request(t, method, url+"/opportunities/ocds-1", nil) + if status != http.StatusMethodNotAllowed { + t.Errorf("%s: status %d; want 405", method, status) + } + } + if status := request(t, "POST", url+"/opportunities", nil); status != 405 { + t.Errorf("POST /opportunities: status %d; want 405", status) + } +} + +// A mark is set with PUT and cleared with DELETE, leaving the others, and +// both reply with all the marks. +func TestChangeMark(t *testing.T) { + url := opportunitiesServer(t, 1) + marks := url + "/opportunities/ocds-1/marks/" + + var got types.OpportunityMarks + request(t, "PUT", marks+"interested", &got) + request(t, "PUT", marks+"bid_submitted", &got) + if !got.Interested || !got.BidSubmitted || got.UUID != "ocds-1" { + t.Fatalf("after two PUTs: %+v", got) + } + + request(t, "DELETE", marks+"interested", &got) + if got.Interested || !got.BidSubmitted { + t.Errorf("after DELETE: %+v; want only bid_submitted", got) + } + + // The change is stored, and setting a set mark again is harmless. + request(t, "PUT", marks+"bid_submitted", &got) + var opportunity types.Opportunity + request(t, "GET", url+"/opportunities/ocds-1", &opportunity) + if !opportunity.Marks.BidSubmitted || opportunity.Marks.Interested { + t.Errorf("stored marks: %+v", opportunity.Marks) + } +} + +// Won and lost exclude each other; unknown marks and opportunities are 404. +func TestChangeMarkRefused(t *testing.T) { + url := opportunitiesServer(t, 1) + marks := url + "/opportunities/ocds-1/marks/" + + request(t, "PUT", marks+"won", nil) + status := request(t, "PUT", marks+"lost", nil) + if status != http.StatusConflict { + t.Errorf("lost while won: status %d; want 409", status) + } + status = request(t, "PUT", marks+"shiny", nil) + if status != 404 { + t.Errorf("unknown mark: status %d; want 404", status) + } + status = request(t, "PUT", url+"/opportunities/nope/marks/won", nil) + if status != 404 { + t.Errorf("unknown opportunity: status %d; want 404", status) + } + + // Clearing won makes lost allowed. + request(t, "DELETE", marks+"won", nil) + status = request(t, "PUT", marks+"lost", nil) + if status != 200 { + t.Errorf("lost after clearing won: status %d; want 200", status) + } +} + +// Concurrent changes to different marks of one opportunity all stick. +func TestChangeMarkConcurrent(t *testing.T) { + url := opportunitiesServer(t, 1) + names := []string{"hidden", "skipped", "interested", "bid_submitted", "won"} + + var wg sync.WaitGroup + for _, name := range names { + wg.Add(1) + go func() { + defer wg.Done() + request(t, "PUT", url+"/opportunities/ocds-1/marks/"+name, nil) + }() + } + wg.Wait() + + var opportunity types.Opportunity + request(t, "GET", url+"/opportunities/ocds-1", &opportunity) + m := opportunity.Marks + if !m.Hidden || !m.Skipped || !m.Interested || !m.BidSubmitted || !m.Won { + t.Errorf("marks after concurrent PUTs: %+v; want all five set", m) + } +} + +// The marks are OpportunityMarks' bool fields, by JSON name, and setting one +// by name sets only that field. +func TestFindMarks(t *testing.T) { + want := "hidden,skipped,interested,bid_submitted,won,lost" + if got := strings.Join(markNames, ","); got != want { + t.Fatalf("markNames = %s; want %s", got, want) + } + + for _, name := range markNames { + var marks types.OpportunityMarks + if !setMark(&marks, name, true) { + t.Fatalf("setMark(%q) = false", name) + } + encoded, err := json.Marshal(marks) + if err != nil { + t.Fatal(err) + } + var fields map[string]any + json.Unmarshal(encoded, &fields) + for field, value := range fields { + if set := value == true; set != (field == name) { + t.Errorf("setting %s: %s = %v", name, field, value) + } + } + } + + var marks types.OpportunityMarks + if setMark(&marks, "uuid", true) || setMark(&marks, "shiny", true) { + t.Error("setMark accepted a name that isn't a mark") + } +} diff --git a/internal/cli/serve/command.go b/internal/cli/serve/command.go index ad812d6..c583c46 100644 --- a/internal/cli/serve/command.go +++ b/internal/cli/serve/command.go @@ -34,7 +34,8 @@ It also runs a worker for every enabled source, as fetcher service does, saving what they fetch to database.file if it is set, and pushing an OpportunitiesUpdated event to every WebSocket client when a save stores new opportunities or new releases. A source that stops with an error stops the -server too, and the server stopping stops the sources.`, +server too, and the server stopping stops the sources. With database.file +set, the API serves the stored opportunities under /api/opportunities.`, PersistentPreRun: func(cmd *cobra.Command, args []string) { slog.Debug("command persistent pre-run", "command", "serve") }, @@ -112,21 +113,15 @@ server too, and the server stopping stops the sources.`, stop() }() - appServer, err := server.New(&conf.Server, options...) - if err != nil { - return err - } - - // runCtx stops whichever of the server and the sources is still - // running once the other has stopped. - runCtx, cancelRun := context.WithCancel(ctx) - defer cancelRun() - + // The store is shared by the sources, which save to it, and the API, + // which serves it. var fetcherOptions []fetcher.Option opportunityStore, openErr := cli.OpenConfiguredStore(ctx, conf) switch { case errors.Is(openErr, cli.ErrNoDatabase): - slog.Warn("not storing fetched opportunities", "reason", openErr) + slog.Warn( + "opportunities aren't stored or served", "reason", openErr, + ) case openErr != nil: return openErr default: @@ -136,8 +131,21 @@ server too, and the server stopping stops the sources.`, fetcherOptions = append( fetcherOptions, fetcher.WithStore(opportunityStore), ) + options = append( + options, server.WithStore(opportunityStore), + ) } + appServer, err := server.New(&conf.Server, options...) + if err != nil { + return err + } + + // runCtx stops whichever of the server and the sources is still + // running once the other has stopped. + runCtx, cancelRun := context.WithCancel(ctx) + defer cancelRun() + // Events from the sources, such as new opportunities, are pushed to // every WebSocket client. var bus events.Bus diff --git a/internal/server/routes.go b/internal/server/routes.go index 979c7f0..8aaa6fe 100644 --- a/internal/server/routes.go +++ b/internal/server/routes.go @@ -60,6 +60,7 @@ func (s *Server) routes() (http.Handler, error) { if s.csrf.TokenEnabled() { apiRouter.Get("/csrf", s.csrf.IssueHandler) } + s.api.Routes(apiRouter) }) // WebSocket (no compression middleware: permessage-deflate is separate) diff --git a/internal/server/server.go b/internal/server/server.go index ae0f534..262e847 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -17,9 +17,11 @@ import ( "github.com/pires/go-proxyproto" + "git.avsdev.uk/Go/job-search/internal/api" "git.avsdev.uk/Go/job-search/internal/config" "git.avsdev.uk/Go/job-search/internal/logging" "git.avsdev.uk/Go/job-search/internal/middleware" + "git.avsdev.uk/Go/job-search/internal/store" "git.avsdev.uk/Go/job-search/internal/util" "git.avsdev.uk/Go/job-search/internal/websocket" ) @@ -61,6 +63,15 @@ func WithAccessLog(logger *log.Logger) Option { } } +// WithStore makes the server's API serve the opportunities in `store` under +// /api/opportunities (see api.WithStore). Without it there are no +// opportunity routes. +func WithStore(store *store.Store) Option { + return func(s *Server) { + s.store = store + } +} + // WithMiddleware appends middleware to a group. This is the generic hook // (auth, metrics, tracing, custom header rules, ...). func WithMiddleware(group Group, middlewares ...Middleware) Option { @@ -107,8 +118,12 @@ type Server struct { proxyProtocolTrusted []netip.Prefix csrf *middleware.CSRF hub *websocket.Hub - started time.Time - handler http.Handler + // store is the opportunities' database, or nil for none. + store *store.Store + // api is the REST API mounted under /api, built from the options. + api *api.API + started time.Time + handler http.Handler // backgroundCtx is for background work, such as the rate limiters' // janitors; backgroundCancel stops it. backgroundCtx context.Context @@ -127,6 +142,7 @@ func New(conf *config.JSCServer, opts ...Option) (*Server, error) { for _, option := range opts { option(server) } + server.api = api.New(api.WithStore(server.store)) var err error server.trusted, err = util.ParseNetPrefixes(conf.Proxy.TrustedProxies) diff --git a/internal/server/server_test.go b/internal/server/server_test.go index 1d3b50b..26c3dff 100644 --- a/internal/server/server_test.go +++ b/internal/server/server_test.go @@ -9,6 +9,7 @@ import ( "net/http" "net/http/cookiejar" "net/http/httptest" + "path/filepath" "strings" "sync" "testing" @@ -20,6 +21,8 @@ import ( "git.avsdev.uk/Go/job-search/internal/config" "git.avsdev.uk/Go/job-search/internal/middleware" + "git.avsdev.uk/Go/job-search/internal/store" + "git.avsdev.uk/Go/job-search/internal/types" ws "git.avsdev.uk/Go/job-search/internal/websocket" ) @@ -87,6 +90,31 @@ func clientWithJar(t *testing.T) *http.Client { } } +// apiClient returns a function making requests to the test server ts with +// client c, sending body as JSON and, if withToken, the CSRF token tok. +func apiClient( + t *testing.T, ts *httptest.Server, c *http.Client, tok csrfToken, +) func(method, path, body string, withToken bool) *http.Response { + t.Helper() + return func(method, path, body string, withToken bool) *http.Response { + req, _ := http.NewRequest(method, ts.URL+path, strings.NewReader(body)) + if body != "" { + req.Header.Set("Content-Type", "application/json") + } + if withToken { + req.Header.Set(tok.Header, tok.Token) + } + res, err := c.Do(req) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + res.Body.Close() + }) + return res + } +} + // Unsafe API requests need the CSRF token: without it they are rejected // before routing; with it they reach the router (here, 405 from a GET-only // route). @@ -307,3 +335,61 @@ func (b *syncBuffer) String() string { defer b.mu.Unlock() return b.buffer.String() } + +// With WithOpportunityStore the opportunities are served, and changing a mark +// needs the CSRF token; without it there are no opportunity routes. +func TestOpportunityRoutes(t *testing.T) { + opportunityStore, err := store.Open( + context.Background(), + filepath.Join(t.TempDir(), "server.db"), + store.WithAutoMigrate(), + ) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + opportunityStore.Close() + }) + _, err = opportunityStore.SaveOpportunity( + context.Background(), + &types.Opportunity{ + UUID: "ocds-1", + UUIDType: "ocid", + Releases: []types.OpportunityRelease{ + { + ReleaseID: "r1", + ReleaseDate: time.Now(), + JSON: "{}", + }, + }, + }, + ) + if err != nil { + t.Fatal(err) + } + + ts := newTestServer(t, nil, WithStore(opportunityStore)) + c := clientWithJar(t) + do := apiClient(t, ts, c, getToken(t, c, ts.URL)) + + if res := do("GET", "/api/opportunities", "", false); res.StatusCode != 200 { + t.Errorf("list: %d; want 200", res.StatusCode) + } + mark := "/api/opportunities/ocds-1/marks/interested" + if res := do("PUT", mark, "", false); res.StatusCode != http.StatusForbidden { + t.Errorf("mark without token: %d; want 403", res.StatusCode) + } + if res := do("PUT", mark, "", true); res.StatusCode != 200 { + t.Errorf("mark with token: %d; want 200", res.StatusCode) + } + + without := newTestServer(t, nil) + res, err := http.Get(without.URL + "/api/opportunities") + if err != nil { + t.Fatal(err) + } + res.Body.Close() + if res.StatusCode != http.StatusNotFound { + t.Errorf("no store: %d; want 404", res.StatusCode) + } +} diff --git a/internal/types/opportunities.go b/internal/types/opportunities.go index c233890..c56f0ec 100644 --- a/internal/types/opportunities.go +++ b/internal/types/opportunities.go @@ -58,7 +58,8 @@ var ErrWonAndLost = errors.New("types: an opportunity cannot be both won and los // OpportunityMarks are the user's own marks on an opportunity, kept apart // from the fetched data so saving a newer release never changes them. The // marks are independent except that Won and Lost exclude each other (see -// Validate). +// Validate). Every bool field is a mark, and the API sets and clears it by +// its JSON name, so a field that isn't a mark mustn't be a bool. type OpportunityMarks struct { UUID string `db:"uuid" json:"uuid"` Hidden bool `db:"hidden" json:"hidden"`