Complete package for generating ocds structs (with extensions) in golang

internal/generator/generator.go

@@ -0,0 +1,351 @@
+// Package generator builds Go types for OCDS releases from the core schema
+// plus a set of extensions. It is used by the ocdsgen command and has no
+// knowledge of the command line.
+package generator
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"os"
+	"path"
+	"path/filepath"
+	"sort"
+	"strings"
+	"time"
+)
+
+// Config is the contents of an extensions.json file.
+type Config struct {
+	ReleaseSchema        string   `json:"releaseSchema"`
+	ReleasePackageSchema string   `json:"releasePackageSchema"`
+	Extensions           []string `json:"extensions"`
+}
+
+// LoadConfig reads and validates a config file. Entries that are relative
+// local paths rather than URLs are resolved against the config file's directory.
+func LoadConfig(p string) (*Config, error) {
+	raw, err := os.ReadFile(p)
+	if err != nil {
+		return nil, err
+	}
+	var cfg Config
+	if err := json.Unmarshal(raw, &cfg); err != nil {
+		return nil, fmt.Errorf("%s: %w", p, err)
+	}
+	if cfg.ReleaseSchema == "" || cfg.ReleasePackageSchema == "" {
+		return nil, fmt.Errorf("%s: releaseSchema and releasePackageSchema are required", p)
+	}
+	dir := filepath.Dir(p)
+	cfg.ReleaseSchema = relativeTo(dir, cfg.ReleaseSchema)
+	cfg.ReleasePackageSchema = relativeTo(dir, cfg.ReleasePackageSchema)
+	for i, e := range cfg.Extensions {
+		cfg.Extensions[i] = relativeTo(dir, e)
+	}
+	return &cfg, nil
+}
+
+func relativeTo(dir, ref string) string {
+	if strings.Contains(ref, "://") || filepath.IsAbs(ref) {
+		return ref
+	}
+	return filepath.Join(dir, ref)
+}
+
+// Extension is one resolved extension.
+type Extension struct {
+	URL          string
+	Name         string
+	Schemas      []string
+	Dependencies []string
+	Declared     bool // listed in the config rather than pulled in as a dependency
+}
+
+// Schemas holds the core schemas after extension patches are applied.
+type Schemas struct {
+	Release *Obj
+	Package *Obj
+}
+
+// Generator holds shared state for a run: an HTTP client, a fetch cache and
+// where warnings go. The zero value is not usable; call New.
+type Generator struct {
+	client   *http.Client
+	cache    map[string][]byte
+	log      io.Writer
+	warnings int
+}
+
+// New returns a Generator that writes notes and warnings to log.
+func New(log io.Writer, timeout time.Duration) *Generator {
+	if log == nil {
+		log = io.Discard
+	}
+	return &Generator{client: &http.Client{Timeout: timeout}, cache: map[string][]byte{}, log: log}
+}
+
+// Warnings reports how many warnings have been emitted so far.
+func (g *Generator) Warnings() int { return g.warnings }
+
+func (g *Generator) warnf(format string, args ...any) {
+	g.warnings++
+	fmt.Fprintf(g.log, "warning: "+format+"\n", args...)
+}
+
+func (g *Generator) notef(format string, args ...any) {
+	fmt.Fprintf(g.log, "note: "+format+"\n", args...)
+}
+
+// Run resolves, merges and emits in one step.
+func (g *Generator) Run(cfg *Config, pkg, configName string, excludeFieldDoc bool) ([]byte, error) {
+	exts, err := g.Resolve(cfg.Extensions)
+	if err != nil {
+		return nil, err
+	}
+	s, err := g.Merge(cfg, exts)
+	if err != nil {
+		return nil, err
+	}
+	return g.Emit(pkg, configName, s, exts, cfg.Extensions, excludeFieldDoc)
+}
+
+// Resolve fetches each extension's metadata and walks dependencies
+// depth-first, so every extension comes after the extensions it depends on.
+// Each URL appears once.
+func (g *Generator) Resolve(urls []string) ([]*Extension, error) {
+	var order []*Extension
+	seen := map[string]bool{}
+	visiting := map[string]bool{}
+	declared := map[string]bool{}
+	for _, u := range urls {
+		declared[u] = true
+	}
+
+	var visit func(u string) error
+	visit = func(u string) error {
+		if seen[u] {
+			return nil
+		}
+		if visiting[u] {
+			return fmt.Errorf("dependency cycle at %s", u)
+		}
+		visiting[u] = true
+
+		var meta struct {
+			Name         map[string]string `json:"name"`
+			Schemas      []string          `json:"schemas"`
+			Dependencies []string          `json:"dependencies"`
+		}
+		body, err := g.fetch(u)
+		if err != nil {
+			return err
+		}
+		if err := json.Unmarshal(body, &meta); err != nil {
+			return fmt.Errorf("%s: %w", u, err)
+		}
+		e := &Extension{
+			URL:          u,
+			Name:         meta.Name["en"],
+			Schemas:      meta.Schemas,
+			Dependencies: meta.Dependencies,
+			Declared:     declared[u],
+		}
+		if e.Name == "" {
+			e.Name = u
+		}
+		for i, d := range meta.Dependencies {
+			// Dependencies are usually absolute URLs; relative ones are
+			// resolved against the extension that declares them.
+			if d, err = resolveRef(u, d); err != nil {
+				return err
+			}
+			e.Dependencies[i] = d
+			if err := visit(d); err != nil {
+				return fmt.Errorf("%s -> %w", e.Name, err)
+			}
+		}
+		visiting[u] = false
+		seen[u] = true
+		order = append(order, e)
+		return nil
+	}
+
+	for _, u := range urls {
+		if err := visit(u); err != nil {
+			return nil, err
+		}
+	}
+	g.warnDuplicateRepos(order)
+	return order, nil
+}
+
+// Merge fetches the core schemas and applies each extension's patches in
+// order using JSON Merge Patch, as the OCDS tooling does.
+func (g *Generator) Merge(cfg *Config, exts []*Extension) (*Schemas, error) {
+	release, err := g.fetchObj(cfg.ReleaseSchema)
+	if err != nil {
+		return nil, err
+	}
+	pkg, err := g.fetchObj(cfg.ReleasePackageSchema)
+	if err != nil {
+		return nil, err
+	}
+	for _, e := range exts {
+		for _, s := range e.Schemas {
+			var target **Obj
+			switch path.Base(s) {
+			case "release-schema.json":
+				target = &release
+			case "release-package-schema.json":
+				target = &pkg
+			default:
+				g.notef("%s: skipping %s (not generated)", e.Name, s)
+				continue
+			}
+			schemaURL, err := resolveRef(e.URL, s)
+			if err != nil {
+				return nil, err
+			}
+			patch, err := g.fetchObj(schemaURL)
+			if err != nil {
+				return nil, fmt.Errorf("%s: %w", e.Name, err)
+			}
+			g.warnTypeConflicts(e.Name, *target, patch)
+			*target = mergePatch(*target, patch).(*Obj)
+		}
+	}
+	return &Schemas{Release: release, Package: pkg}, nil
+}
+
+// warnDuplicateRepos flags the same extension pulled in at two different
+// versions (e.g. .../master/extension.json and .../v1.1.5/extension.json).
+func (g *Generator) warnDuplicateRepos(exts []*Extension) {
+	byRepo := map[string][]string{}
+	for _, e := range exts {
+		u, err := url.Parse(e.URL)
+		if err != nil {
+			continue
+		}
+		parts := strings.Split(strings.Trim(u.Path, "/"), "/")
+		if len(parts) >= 2 {
+			repo := strings.ToLower(parts[0] + "/" + parts[1])
+			byRepo[repo] = append(byRepo[repo], e.URL)
+		}
+	}
+	repos := make([]string, 0, len(byRepo))
+	for r := range byRepo {
+		repos = append(repos, r)
+	}
+	sort.Strings(repos)
+	for _, r := range repos {
+		if len(byRepo[r]) > 1 {
+			g.warnf("%s is included more than once: %s", r, strings.Join(byRepo[r], ", "))
+		}
+	}
+}
+
+// warnTypeConflicts reports when a patch changes the JSON type of an existing
+// property, which usually means two extensions disagree.
+func (g *Generator) warnTypeConflicts(ext string, target, patch *Obj) {
+	check := func(where string, tProps, pProps *Obj) {
+		if tProps == nil || pProps == nil {
+			return
+		}
+		for _, k := range pProps.Keys {
+			pp, _ := pProps.M[k].(*Obj)
+			tp, _ := tProps.M[k].(*Obj)
+			if pp == nil || tp == nil {
+				continue
+			}
+			if a, b := typeKey(tp), typeKey(pp); a != "" && b != "" && a != b {
+				g.warnf("%s changes %s.%s from %s to %s", ext, where, k, a, b)
+			}
+		}
+	}
+	check("Release", target.Obj("properties"), patch.Obj("properties"))
+	pDefs, tDefs := patch.Obj("definitions"), target.Obj("definitions")
+	if pDefs == nil || tDefs == nil {
+		return
+	}
+	for _, name := range pDefs.Keys {
+		if pd, td := pDefs.Obj(name), tDefs.Obj(name); pd != nil && td != nil {
+			check(name, td.Obj("properties"), pd.Obj("properties"))
+		}
+	}
+}
+
+func typeKey(s *Obj) string {
+	if ref := s.Str("$ref"); ref != "" {
+		return "$ref:" + ref
+	}
+	ts := typeSet(s)
+	sort.Strings(ts)
+	return strings.Join(ts, "|")
+}
+
+func resolveRef(base, ref string) (string, error) {
+	if !strings.Contains(base, "://") {
+		// Local path: URL resolution would turn "a/b.json" + "../c.json"
+		// into "/c.json", so use file path rules instead.
+		if strings.Contains(ref, "://") || filepath.IsAbs(ref) {
+			return ref, nil
+		}
+		return filepath.Join(filepath.Dir(base), ref), nil
+	}
+	b, err := url.Parse(base)
+	if err != nil {
+		return "", err
+	}
+	r, err := url.Parse(ref)
+	if err != nil {
+		return "", err
+	}
+	return b.ResolveReference(r).String(), nil
+}
+
+func (g *Generator) fetch(u string) ([]byte, error) {
+	if b, ok := g.cache[u]; ok {
+		return b, nil
+	}
+	var body []byte
+	if strings.HasPrefix(u, "http://") || strings.HasPrefix(u, "https://") {
+		resp, err := g.client.Get(u)
+		if err != nil {
+			return nil, err
+		}
+		defer resp.Body.Close()
+		if resp.StatusCode != http.StatusOK {
+			return nil, fmt.Errorf("GET %s: %s", u, resp.Status)
+		}
+		if body, err = io.ReadAll(resp.Body); err != nil {
+			return nil, err
+		}
+	} else {
+		// Local paths are allowed, e.g. for a vendored or private extension.
+		var err error
+		if body, err = os.ReadFile(strings.TrimPrefix(u, "file://")); err != nil {
+			return nil, err
+		}
+	}
+	g.cache[u] = body
+	return body, nil
+}
+
+func (g *Generator) fetchObj(u string) (*Obj, error) {
+	body, err := g.fetch(u)
+	if err != nil {
+		return nil, err
+	}
+	v, err := decodeOrdered(bytes.NewReader(body))
+	if err != nil {
+		return nil, fmt.Errorf("%s: %w", u, err)
+	}
+	o, ok := v.(*Obj)
+	if !ok {
+		return nil, fmt.Errorf("%s: not a JSON object", u)
+	}
+	return o, nil
+}