go-ocds/internal/generator/generator.go

// Package generator builds Go types for OCDS releases from the core schema
// plus a set of extensions. It is used by the ocdsgen command and has no
// knowledge of the command line.
package generator

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"net/http"
	"net/url"
	"os"
	"path"
	"path/filepath"
	"sort"
	"strings"
	"time"
)

// Config is the contents of an extensions.json file.
type Config struct {
	ReleaseSchema        string   `json:"releaseSchema"`
	ReleasePackageSchema string   `json:"releasePackageSchema"`
	Extensions           []string `json:"extensions"`
}

// LoadConfig reads and validates a config file. Entries that are relative
// local paths rather than URLs are resolved against the config file's directory.
func LoadConfig(p string) (*Config, error) {
	raw, err := os.ReadFile(p)
	if err != nil {
		return nil, err
	}
	var cfg Config
	if err := json.Unmarshal(raw, &cfg); err != nil {
		return nil, fmt.Errorf("%s: %w", p, err)
	}
	if cfg.ReleaseSchema == "" || cfg.ReleasePackageSchema == "" {
		return nil, fmt.Errorf("%s: releaseSchema and releasePackageSchema are required", p)
	}
	dir := filepath.Dir(p)
	cfg.ReleaseSchema = relativeTo(dir, cfg.ReleaseSchema)
	cfg.ReleasePackageSchema = relativeTo(dir, cfg.ReleasePackageSchema)
	for i, e := range cfg.Extensions {
		cfg.Extensions[i] = relativeTo(dir, e)
	}
	return &cfg, nil
}

func relativeTo(dir, ref string) string {
	if strings.Contains(ref, "://") || filepath.IsAbs(ref) {
		return ref
	}
	return filepath.Join(dir, ref)
}

// Extension is one resolved extension.
type Extension struct {
	URL          string
	Name         string
	Schemas      []string
	Dependencies []string
	Declared     bool // listed in the config rather than pulled in as a dependency
}

// Schemas holds the core schemas after extension patches are applied.
type Schemas struct {
	Release *Obj
	Package *Obj
}

// Generator holds shared state for a run: an HTTP client, a fetch cache and
// where warnings go. The zero value is not usable; call New.
type Generator struct {
	client   *http.Client
	cache    map[string][]byte
	log      io.Writer
	warnings int
}

// New returns a Generator that writes notes and warnings to log.
func New(log io.Writer, timeout time.Duration) *Generator {
	if log == nil {
		log = io.Discard
	}
	return &Generator{client: &http.Client{Timeout: timeout}, cache: map[string][]byte{}, log: log}
}

// Warnings reports how many warnings have been emitted so far.
func (g *Generator) Warnings() int { return g.warnings }

func (g *Generator) warnf(format string, args ...any) {
	g.warnings++
	fmt.Fprintf(g.log, "warning: "+format+"\n", args...)
}

func (g *Generator) notef(format string, args ...any) {
	fmt.Fprintf(g.log, "note: "+format+"\n", args...)
}

// Run resolves, merges and emits in one step.
func (g *Generator) Run(cfg *Config, pkg, configName string, excludeFieldDoc bool) ([]byte, error) {
	exts, err := g.Resolve(cfg.Extensions)
	if err != nil {
		return nil, err
	}
	s, err := g.Merge(cfg, exts)
	if err != nil {
		return nil, err
	}
	return g.Emit(pkg, configName, s, exts, cfg.Extensions, excludeFieldDoc)
}

// Resolve fetches each extension's metadata and walks dependencies
// depth-first, so every extension comes after the extensions it depends on.
// Each URL appears once.
func (g *Generator) Resolve(urls []string) ([]*Extension, error) {
	var order []*Extension
	seen := map[string]bool{}
	visiting := map[string]bool{}
	declared := map[string]bool{}
	for _, u := range urls {
		declared[u] = true
	}

	var visit func(u string) error
	visit = func(u string) error {
		if seen[u] {
			return nil
		}
		if visiting[u] {
			return fmt.Errorf("dependency cycle at %s", u)
		}
		visiting[u] = true

		var meta struct {
			Name         map[string]string `json:"name"`
			Schemas      []string          `json:"schemas"`
			Dependencies []string          `json:"dependencies"`
		}
		body, err := g.fetch(u)
		if err != nil {
			return err
		}
		if err := json.Unmarshal(body, &meta); err != nil {
			return fmt.Errorf("%s: %w", u, err)
		}
		e := &Extension{
			URL:          u,
			Name:         meta.Name["en"],
			Schemas:      meta.Schemas,
			Dependencies: meta.Dependencies,
			Declared:     declared[u],
		}
		if e.Name == "" {
			e.Name = u
		}
		for i, d := range meta.Dependencies {
			// Dependencies are usually absolute URLs; relative ones are
			// resolved against the extension that declares them.
			if d, err = resolveRef(u, d); err != nil {
				return err
			}
			e.Dependencies[i] = d
			if err := visit(d); err != nil {
				return fmt.Errorf("%s -> %w", e.Name, err)
			}
		}
		visiting[u] = false
		seen[u] = true
		order = append(order, e)
		return nil
	}

	for _, u := range urls {
		if err := visit(u); err != nil {
			return nil, err
		}
	}
	g.warnDuplicateRepos(order)
	return order, nil
}

// Merge fetches the core schemas and applies each extension's patches in
// order using JSON Merge Patch, as the OCDS tooling does.
func (g *Generator) Merge(cfg *Config, exts []*Extension) (*Schemas, error) {
	release, err := g.fetchObj(cfg.ReleaseSchema)
	if err != nil {
		return nil, err
	}
	pkg, err := g.fetchObj(cfg.ReleasePackageSchema)
	if err != nil {
		return nil, err
	}
	for _, e := range exts {
		for _, s := range e.Schemas {
			var target **Obj
			switch path.Base(s) {
			case "release-schema.json":
				target = &release
			case "release-package-schema.json":
				target = &pkg
			default:
				g.notef("%s: skipping %s (not generated)", e.Name, s)
				continue
			}
			schemaURL, err := resolveRef(e.URL, s)
			if err != nil {
				return nil, err
			}
			patch, err := g.fetchObj(schemaURL)
			if err != nil {
				return nil, fmt.Errorf("%s: %w", e.Name, err)
			}
			g.warnTypeConflicts(e.Name, *target, patch)
			*target = mergePatch(*target, patch).(*Obj)
		}
	}
	return &Schemas{Release: release, Package: pkg}, nil
}

// warnDuplicateRepos flags the same extension pulled in at two different
// versions (e.g. .../master/extension.json and .../v1.1.5/extension.json).
func (g *Generator) warnDuplicateRepos(exts []*Extension) {
	byRepo := map[string][]string{}
	for _, e := range exts {
		u, err := url.Parse(e.URL)
		if err != nil {
			continue
		}
		parts := strings.Split(strings.Trim(u.Path, "/"), "/")
		if len(parts) >= 2 {
			repo := strings.ToLower(parts[0] + "/" + parts[1])
			byRepo[repo] = append(byRepo[repo], e.URL)
		}
	}
	repos := make([]string, 0, len(byRepo))
	for r := range byRepo {
		repos = append(repos, r)
	}
	sort.Strings(repos)
	for _, r := range repos {
		if len(byRepo[r]) > 1 {
			g.warnf("%s is included more than once: %s", r, strings.Join(byRepo[r], ", "))
		}
	}
}

// warnTypeConflicts reports when a patch changes the JSON type of an existing
// property, which usually means two extensions disagree.
func (g *Generator) warnTypeConflicts(ext string, target, patch *Obj) {
	check := func(where string, tProps, pProps *Obj) {
		if tProps == nil || pProps == nil {
			return
		}
		for _, k := range pProps.Keys {
			pp, _ := pProps.M[k].(*Obj)
			tp, _ := tProps.M[k].(*Obj)
			if pp == nil || tp == nil {
				continue
			}
			if a, b := typeKey(tp), typeKey(pp); a != "" && b != "" && a != b {
				g.warnf("%s changes %s.%s from %s to %s", ext, where, k, a, b)
			}
		}
	}
	check("Release", target.Obj("properties"), patch.Obj("properties"))
	pDefs, tDefs := patch.Obj("definitions"), target.Obj("definitions")
	if pDefs == nil || tDefs == nil {
		return
	}
	for _, name := range pDefs.Keys {
		if pd, td := pDefs.Obj(name), tDefs.Obj(name); pd != nil && td != nil {
			check(name, td.Obj("properties"), pd.Obj("properties"))
		}
	}
}

func typeKey(s *Obj) string {
	if ref := s.Str("$ref"); ref != "" {
		return "$ref:" + ref
	}
	ts := typeSet(s)
	sort.Strings(ts)
	return strings.Join(ts, "|")
}

func resolveRef(base, ref string) (string, error) {
	if !strings.Contains(base, "://") {
		// Local path: URL resolution would turn "a/b.json" + "../c.json"
		// into "/c.json", so use file path rules instead.
		if strings.Contains(ref, "://") || filepath.IsAbs(ref) {
			return ref, nil
		}
		return filepath.Join(filepath.Dir(base), ref), nil
	}
	b, err := url.Parse(base)
	if err != nil {
		return "", err
	}
	r, err := url.Parse(ref)
	if err != nil {
		return "", err
	}
	return b.ResolveReference(r).String(), nil
}

func (g *Generator) fetch(u string) ([]byte, error) {
	if b, ok := g.cache[u]; ok {
		return b, nil
	}
	var body []byte
	if strings.HasPrefix(u, "http://") || strings.HasPrefix(u, "https://") {
		resp, err := g.client.Get(u)
		if err != nil {
			return nil, err
		}
		defer resp.Body.Close()
		if resp.StatusCode != http.StatusOK {
			return nil, fmt.Errorf("GET %s: %s", u, resp.Status)
		}
		if body, err = io.ReadAll(resp.Body); err != nil {
			return nil, err
		}
	} else {
		// Local paths are allowed, e.g. for a vendored or private extension.
		var err error
		if body, err = os.ReadFile(strings.TrimPrefix(u, "file://")); err != nil {
			return nil, err
		}
	}
	g.cache[u] = body
	return body, nil
}

func (g *Generator) fetchObj(u string) (*Obj, error) {
	body, err := g.fetch(u)
	if err != nil {
		return nil, err
	}
	v, err := decodeOrdered(bytes.NewReader(body))
	if err != nil {
		return nil, fmt.Errorf("%s: %w", u, err)
	}
	o, ok := v.(*Obj)
	if !ok {
		return nil, fmt.Errorf("%s: not a JSON object", u)
	}
	return o, nil
}