Complete package for generating ocds structs (with extensions) in golang

internal/cli/cli_test.go

@@ -0,0 +1,151 @@
+package cli
+
+import (
+	"bytes"
+	"encoding/json"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+)
+
+// run executes the command tree with args and returns stdout, stderr and the error.
+func run(t *testing.T, args ...string) (string, string, error) {
+	t.Helper()
+	var stdout, stderr bytes.Buffer
+	cmd := NewRootCmd()
+	cmd.SetOut(&stdout)
+	cmd.SetErr(&stderr)
+	cmd.SetArgs(args)
+	err := cmd.Execute()
+	return stdout.String(), stderr.String(), err
+}
+
+func TestGenerate(t *testing.T) {
+	out := filepath.Join(t.TempDir(), "tenders", "ocds_gen.go")
+	if err := os.MkdirAll(filepath.Dir(out), 0o755); err != nil {
+		t.Fatal(err)
+	}
+	if _, stderr, err := run(t, "generate", "-c", "testdata/widgets.json", "-o", out); err != nil {
+		t.Fatalf("generate: %v\n%s", err, stderr)
+	}
+	raw, err := os.ReadFile(out)
+	if err != nil {
+		t.Fatal(err)
+	}
+	// gofmt aligns struct columns, so compare with whitespace collapsed.
+	src := strings.Join(strings.Fields(string(raw)), " ")
+	for _, want := range []string{
+		"package tenders",     // derived from the output directory
+		"Date time.Time",      // required date-time is not a pointer
+		"ID ID `json:\"id\"`", // ["string","integer"]
+		"Widgets []Widget",    // extension field on a core type
+		"Count *int",          // extension definition
+		"Releases []Release",  // external $ref to the release schema
+		"Widgets (declared)",  // header records resolution
+		"Orphan (dependency)",
+	} {
+		if !strings.Contains(src, want) {
+			t.Errorf("generated code is missing %q", want)
+		}
+	}
+}
+
+func TestGenerateStdoutNeedsPackage(t *testing.T) {
+	_, _, err := run(t, "generate", "-c", "testdata/core.json", "-o", "-")
+	if err == nil || !strings.Contains(err.Error(), "--package") {
+		t.Fatalf("want --package error, got %v", err)
+	}
+	stdout, _, err := run(t, "generate", "-c", "testdata/core.json", "-o", "-", "-p", "core")
+	if err != nil || !strings.HasPrefix(stdout, "// Code generated") {
+		t.Fatalf("stdout generation failed: %v", err)
+	}
+}
+
+func TestCheck(t *testing.T) {
+	out := filepath.Join(t.TempDir(), "ocds_gen.go")
+	args := []string{"generate", "-q", "-c", "testdata/core.json", "-o", out, "-p", "core"}
+
+	if _, _, err := run(t, append(args, "--check")...); err == nil || !strings.Contains(err.Error(), "does not exist") {
+		t.Fatalf("missing file: want error, got %v", err)
+	}
+	if _, _, err := run(t, args...); err != nil {
+		t.Fatal(err)
+	}
+	if _, _, err := run(t, append(args, "--check")...); err != nil {
+		t.Fatalf("fresh file: want no error, got %v", err)
+	}
+	f, _ := os.OpenFile(out, os.O_APPEND|os.O_WRONLY, 0)
+	f.WriteString("// hand edit\n")
+	f.Close()
+	if _, _, err := run(t, append(args, "--check")...); err == nil || !strings.Contains(err.Error(), "out of date") {
+		t.Fatalf("edited file: want out-of-date error, got %v", err)
+	}
+}
+
+func TestStrict(t *testing.T) {
+	out := filepath.Join(t.TempDir(), "ocds_gen.go")
+	args := []string{"generate", "-c", "testdata/widgets.json", "-o", out, "-p", "x"}
+
+	// Gadget is defined by the Orphan extension but nothing refers to it.
+	_, stderr, err := run(t, args...)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !strings.Contains(stderr, "definition Gadget is never referenced") {
+		t.Errorf("want unreachable warning, got stderr:\n%s", stderr)
+	}
+	if _, _, err := run(t, append(args, "--strict")...); err == nil {
+		t.Fatal("--strict: want error")
+	}
+	if _, stderr, _ := run(t, append(args, "-q")...); stderr != "" {
+		t.Errorf("--quiet: want no stderr, got:\n%s", stderr)
+	}
+}
+
+func TestResolveJSON(t *testing.T) {
+	stdout, _, err := run(t, "resolve", "-c", "testdata/widgets.json", "--json")
+	if err != nil {
+		t.Fatal(err)
+	}
+	var rows []struct {
+		Name     string `json:"name"`
+		Declared bool   `json:"declared"`
+	}
+	if err := json.Unmarshal([]byte(stdout), &rows); err != nil {
+		t.Fatal(err)
+	}
+	if len(rows) != 2 || rows[0].Name != "Orphan" || rows[0].Declared || rows[1].Name != "Widgets" || !rows[1].Declared {
+		t.Fatalf("want [Orphan (dependency), Widgets (declared)], got %+v", rows)
+	}
+}
+
+func TestSchema(t *testing.T) {
+	stdout, _, err := run(t, "schema", "-q", "-c", "testdata/widgets.json")
+	if err != nil {
+		t.Fatal(err)
+	}
+	// Key order is preserved, and the extension's patch is merged in.
+	if !strings.Contains(stdout, `"widgets"`) || strings.Index(stdout, `"ocid"`) > strings.Index(stdout, `"tender"`) {
+		t.Fatalf("unexpected merged schema:\n%s", stdout)
+	}
+	if _, _, err := run(t, "schema", "--kind", "record", "-c", "testdata/core.json"); err == nil {
+		t.Fatal("--kind record: want error")
+	}
+}
+
+func TestPackageFromPath(t *testing.T) {
+	for in, want := range map[string]string{
+		"internal/tenders/ocds_gen.go": "tenders",
+		"profiles/uk/ocds_gen.go":      "uk",
+		"pkg/eu-profile/x.go":          "euprofile",
+	} {
+		got, err := packageFromPath(in)
+		if err != nil || got != want {
+			t.Errorf("packageFromPath(%q) = %q, %v; want %q", in, got, err, want)
+		}
+	}
+	if _, err := packageFromPath("pkg/func/x.go"); err == nil {
+		t.Error("keyword directory: want error")
+	}
+}

internal/cli/generate.go

@@ -0,0 +1,106 @@
+package cli
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"go/token"
+	"os"
+	"path/filepath"
+	"strings"
+	"unicode"
+
+	"github.com/spf13/cobra"
+)
+
+func newGenerateCmd(opts *options) *cobra.Command {
+	var (
+		out             string
+		pkg             string
+		check           bool
+		excludeFieldDoc bool
+	)
+	cmd := &cobra.Command{
+		Use:   "generate",
+		Short: "Generate Go types from the config",
+		Long: `Generate fetches the schemas and extensions named in the config and writes
+the Go types to --out. Use --out - to write to stdout.
+
+With --check, nothing is written; the command fails if the output file is
+missing or differs from what would be generated. Use this in CI to catch
+generated code that is stale or was edited by hand.`,
+		Example: `  ocdsgen generate -c extensions.json -o ocds_gen.go -p tenders
+  ocdsgen generate --check --strict
+  //go:generate go tool ocdsgen generate -c extensions.json -o ocds_gen.go -p tenders`,
+		Args: cobra.NoArgs,
+		RunE: func(cmd *cobra.Command, _ []string) error {
+			if pkg == "" {
+				var err error
+				if pkg, err = packageFromPath(out); err != nil {
+					return err
+				}
+			}
+			cfg, g, err := opts.setup(cmd)
+			if err != nil {
+				return err
+			}
+			src, err := g.Run(cfg, pkg, filepath.Base(opts.config), excludeFieldDoc)
+			if err != nil {
+				return err
+			}
+
+			switch {
+			case out == "-":
+				if _, err := cmd.OutOrStdout().Write(src); err != nil {
+					return err
+				}
+			case check:
+				existing, err := os.ReadFile(out)
+				if errors.Is(err, os.ErrNotExist) {
+					return fmt.Errorf("%s does not exist; run ocdsgen generate", out)
+				} else if err != nil {
+					return err
+				}
+				if !bytes.Equal(existing, src) {
+					return fmt.Errorf("%s is out of date; run ocdsgen generate", out)
+				}
+				fmt.Fprintf(cmd.ErrOrStderr(), "%s is up to date\n", out)
+			default:
+				if err := os.WriteFile(out, src, 0o644); err != nil {
+					return err
+				}
+			}
+			return opts.finish(g)
+		},
+	}
+
+	f := cmd.Flags()
+	f.StringVarP(&out, "out", "o", "ocds_gen.go", `output Go file, or "-" for stdout`)
+	f.StringVarP(&pkg, "package", "p", "", "Go package name (default: name of the output file's directory)")
+	f.BoolVar(&check, "check", false, "verify the output file is up to date instead of writing it")
+	f.BoolVar(&excludeFieldDoc, "exclude-field-doc", false, `exclude documentation for definition fields`)
+	return cmd
+}
+
+// packageFromPath derives a package name from the directory the output file
+// will be written to, e.g. internal/tenders/ocds_gen.go -> tenders.
+func packageFromPath(out string) (string, error) {
+	if out == "-" {
+		return "", errors.New("--package is required when writing to stdout")
+	}
+	abs, err := filepath.Abs(out)
+	if err != nil {
+		return "", err
+	}
+	dir := filepath.Base(filepath.Dir(abs))
+	name := strings.Map(func(r rune) rune {
+		if unicode.IsLetter(r) || unicode.IsDigit(r) || r == '_' {
+			return unicode.ToLower(r)
+		}
+		return -1
+	}, dir)
+	if !token.IsIdentifier(name) || token.IsKeyword(name) {
+		return "", fmt.Errorf("can't derive a package name from directory %q; pass --package", dir)
+	}
+	return name, nil
+}

internal/cli/resolve.go

@@ -0,0 +1,65 @@
+package cli
+
+import (
+	"encoding/json"
+	"fmt"
+	"text/tabwriter"
+
+	"github.com/spf13/cobra"
+)
+
+func newResolveCmd(opts *options) *cobra.Command {
+	var asJSON bool
+	cmd := &cobra.Command{
+		Use:   "resolve",
+		Short: "List the extensions the config resolves to, in merge order",
+		Long: `Resolve fetches each extension in the config and its dependencies, and prints
+them in the order their patches are applied. Dependencies come first. Nothing
+is generated.`,
+		Args: cobra.NoArgs,
+		RunE: func(cmd *cobra.Command, _ []string) error {
+			cfg, g, err := opts.setup(cmd)
+			if err != nil {
+				return err
+			}
+			exts, err := g.Resolve(cfg.Extensions)
+			if err != nil {
+				return err
+			}
+
+			if asJSON {
+				type row struct {
+					Name         string   `json:"name"`
+					URL          string   `json:"url"`
+					Declared     bool     `json:"declared"`
+					Dependencies []string `json:"dependencies,omitempty"`
+				}
+				rows := make([]row, len(exts))
+				for i, e := range exts {
+					rows[i] = row{e.Name, e.URL, e.Declared, e.Dependencies}
+				}
+				enc := json.NewEncoder(cmd.OutOrStdout())
+				enc.SetIndent("", "  ")
+				if err := enc.Encode(rows); err != nil {
+					return err
+				}
+			} else {
+				w := tabwriter.NewWriter(cmd.OutOrStdout(), 0, 0, 2, ' ', 0)
+				fmt.Fprintln(w, "#\tNAME\tSOURCE\tURL")
+				for i, e := range exts {
+					src := "dependency"
+					if e.Declared {
+						src = "declared"
+					}
+					fmt.Fprintf(w, "%d\t%s\t%s\t%s\n", i+1, e.Name, src, e.URL)
+				}
+				if err := w.Flush(); err != nil {
+					return err
+				}
+			}
+			return opts.finish(g)
+		},
+	}
+	cmd.Flags().BoolVar(&asJSON, "json", false, "print as JSON")
+	return cmd
+}

internal/cli/root.go

@@ -0,0 +1,72 @@
+// Package cli defines the ocdsgen command tree using Cobra. The commands are
+// thin wrappers around internal/generator.
+package cli
+
+import (
+	"fmt"
+	"io"
+	"time"
+
+	"github.com/spf13/cobra"
+
+	"git.avsdev.uk/Go/go-ocds/internal/generator"
+)
+
+// options are the flags shared by every subcommand.
+type options struct {
+	config  string
+	timeout time.Duration
+	strict  bool
+	quiet   bool
+}
+
+// NewRootCmd builds the command tree. It returns a fresh tree on each call so
+// tests can run commands in isolation.
+func NewRootCmd() *cobra.Command {
+	opts := &options{}
+
+	root := &cobra.Command{
+		Use:   "ocdsgen",
+		Short: "Generate Go types for OCDS releases with chosen extensions",
+		Long: `ocdsgen reads an extensions.json config listing the OCDS core schemas and
+extension URLs, resolves extension dependencies, applies each extension's
+schema patch in order, and generates Go structs from the result.`,
+		SilenceUsage:  true, // don't print usage on runtime errors
+		SilenceErrors: true, // main prints the error once
+	}
+
+	pf := root.PersistentFlags()
+	pf.StringVarP(&opts.config, "config", "c", "extensions.json", "path to the extensions config file")
+	pf.DurationVar(&opts.timeout, "timeout", 60*time.Second, "timeout for each HTTP request")
+	pf.BoolVar(&opts.strict, "strict", false, "fail if the generator reports any warnings")
+	pf.BoolVarP(&opts.quiet, "quiet", "q", false, "don't print notes and warnings")
+
+	root.AddCommand(
+		newGenerateCmd(opts),
+		newResolveCmd(opts),
+		newSchemaCmd(opts),
+	)
+	return root
+}
+
+// setup loads the config and creates a Generator that logs to the command's
+// stderr (or nowhere, with --quiet).
+func (o *options) setup(cmd *cobra.Command) (*generator.Config, *generator.Generator, error) {
+	cfg, err := generator.LoadConfig(o.config)
+	if err != nil {
+		return nil, nil, err
+	}
+	var log io.Writer = cmd.ErrOrStderr()
+	if o.quiet {
+		log = io.Discard
+	}
+	return cfg, generator.New(log, o.timeout), nil
+}
+
+// finish enforces --strict once a command's work is done.
+func (o *options) finish(g *generator.Generator) error {
+	if n := g.Warnings(); o.strict && n > 0 {
+		return fmt.Errorf("%d warning(s) reported and --strict is set", n)
+	}
+	return nil
+}

internal/cli/schema.go

@@ -0,0 +1,66 @@
+package cli
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+
+	"github.com/spf13/cobra"
+)
+
+func newSchemaCmd(opts *options) *cobra.Command {
+	var (
+		kind string
+		out  string
+	)
+	cmd := &cobra.Command{
+		Use:   "schema",
+		Short: "Write the merged JSON Schema (core plus extensions)",
+		Long: `Schema writes the release or release package schema with every extension
+in the config applied. This is the schema the Go types are generated from,
+which is useful for debugging a config or validating data with other tools.`,
+		Example: `  ocdsgen schema > release-schema.json
+  ocdsgen schema --kind package -o release-package-schema.json`,
+		Args: cobra.NoArgs,
+		RunE: func(cmd *cobra.Command, _ []string) error {
+			if kind != "release" && kind != "package" {
+				return fmt.Errorf(`--kind must be "release" or "package", got %q`, kind)
+			}
+			cfg, g, err := opts.setup(cmd)
+			if err != nil {
+				return err
+			}
+			exts, err := g.Resolve(cfg.Extensions)
+			if err != nil {
+				return err
+			}
+			s, err := g.Merge(cfg, exts)
+			if err != nil {
+				return err
+			}
+			doc := s.Release
+			if kind == "package" {
+				doc = s.Package
+			}
+			b, err := json.MarshalIndent(doc, "", "  ")
+			if err != nil {
+				return err
+			}
+			b = append(b, '\n')
+
+			if out == "-" {
+				_, err = cmd.OutOrStdout().Write(b)
+			} else {
+				err = os.WriteFile(out, b, 0o644)
+			}
+			if err != nil {
+				return err
+			}
+			return opts.finish(g)
+		},
+	}
+	f := cmd.Flags()
+	f.StringVar(&kind, "kind", "release", `which schema to write: "release" or "package"`)
+	f.StringVarP(&out, "out", "o", "-", `output file, or "-" for stdout`)
+	return cmd
+}

internal/cli/testdata/core.json

@@ -0,0 +1,5 @@
+{
+  "releaseSchema": "release-schema.json",
+  "releasePackageSchema": "release-package-schema.json",
+  "extensions": []
+}

internal/cli/testdata/orphan/extension.json

@@ -0,0 +1 @@
+{"name": {"en": "Orphan"}, "schemas": ["release-schema.json"]}

internal/cli/testdata/orphan/release-schema.json

@@ -0,0 +1 @@
+{"definitions": {"Gadget": {"type": "object", "properties": {"name": {"type": "string"}}}}}

internal/cli/testdata/release-package-schema.json

@@ -0,0 +1,7 @@
+{
+  "type": "object",
+  "properties": {
+    "uri": {"type": "string", "format": "uri"},
+    "releases": {"type": "array", "items": {"$ref": "https://standard.open-contracting.org/schema/1__1__5/release-schema.json"}}
+  }
+}

internal/cli/testdata/release-schema.json

@@ -0,0 +1,20 @@
+{
+  "type": "object",
+  "required": ["ocid", "id", "date"],
+  "properties": {
+    "ocid": {"type": "string", "description": "Open Contracting ID."},
+    "id": {"type": "string"},
+    "date": {"type": "string", "format": "date-time"},
+    "tender": {"$ref": "#/definitions/Tender"}
+  },
+  "definitions": {
+    "Tender": {
+      "type": "object",
+      "required": ["id"],
+      "properties": {
+        "id": {"type": ["string", "integer"]},
+        "title": {"type": ["string", "null"]}
+      }
+    }
+  }
+}

internal/cli/testdata/widgets.json

@@ -0,0 +1,5 @@
+{
+  "releaseSchema": "release-schema.json",
+  "releasePackageSchema": "release-package-schema.json",
+  "extensions": ["widgets/extension.json"]
+}

internal/cli/testdata/widgets/extension.json

@@ -0,0 +1 @@
+{"name": {"en": "Widgets"}, "schemas": ["release-schema.json"], "dependencies": ["../orphan/extension.json"]}

internal/cli/testdata/widgets/release-schema.json

@@ -0,0 +1,6 @@
+{
+  "definitions": {
+    "Tender": {"properties": {"widgets": {"type": "array", "items": {"$ref": "#/definitions/Widget"}}}},
+    "Widget": {"type": "object", "properties": {"id": {"type": "string"}, "count": {"type": "integer"}}}
+  }
+}