// Package generator builds Go types for OCDS releases from the core schema // plus a set of extensions. It is used by the ocdsgen command and has no // knowledge of the command line. package generator import ( "bytes" "encoding/json" "fmt" "io" "net/http" "net/url" "os" "path" "path/filepath" "sort" "strings" "time" ) // Config is the contents of an extensions.json file. type Config struct { ReleaseSchema string `json:"releaseSchema"` ReleasePackageSchema string `json:"releasePackageSchema"` Extensions []string `json:"extensions"` } // LoadConfig reads and validates a config file. Entries that are relative // local paths rather than URLs are resolved against the config file's directory. func LoadConfig(p string) (*Config, error) { raw, err := os.ReadFile(p) if err != nil { return nil, err } var cfg Config if err := json.Unmarshal(raw, &cfg); err != nil { return nil, fmt.Errorf("%s: %w", p, err) } if cfg.ReleaseSchema == "" || cfg.ReleasePackageSchema == "" { return nil, fmt.Errorf("%s: releaseSchema and releasePackageSchema are required", p) } dir := filepath.Dir(p) cfg.ReleaseSchema = relativeTo(dir, cfg.ReleaseSchema) cfg.ReleasePackageSchema = relativeTo(dir, cfg.ReleasePackageSchema) for i, e := range cfg.Extensions { cfg.Extensions[i] = relativeTo(dir, e) } return &cfg, nil } func relativeTo(dir, ref string) string { if strings.Contains(ref, "://") || filepath.IsAbs(ref) { return ref } return filepath.Join(dir, ref) } // Extension is one resolved extension. type Extension struct { URL string Name string Schemas []string Dependencies []string Declared bool // listed in the config rather than pulled in as a dependency } // Schemas holds the core schemas after extension patches are applied. type Schemas struct { Release *Obj Package *Obj } // Generator holds shared state for a run: an HTTP client, a fetch cache and // where warnings go. The zero value is not usable; call New. type Generator struct { client *http.Client cache map[string][]byte log io.Writer warnings int } // New returns a Generator that writes notes and warnings to log. func New(log io.Writer, timeout time.Duration) *Generator { if log == nil { log = io.Discard } return &Generator{client: &http.Client{Timeout: timeout}, cache: map[string][]byte{}, log: log} } // Warnings reports how many warnings have been emitted so far. func (g *Generator) Warnings() int { return g.warnings } func (g *Generator) warnf(format string, args ...any) { g.warnings++ fmt.Fprintf(g.log, "warning: "+format+"\n", args...) } func (g *Generator) notef(format string, args ...any) { fmt.Fprintf(g.log, "note: "+format+"\n", args...) } // Run resolves, merges and emits in one step. func (g *Generator) Run(cfg *Config, pkg, configName string, excludeFieldDoc bool) ([]byte, error) { exts, err := g.Resolve(cfg.Extensions) if err != nil { return nil, err } s, err := g.Merge(cfg, exts) if err != nil { return nil, err } return g.Emit(pkg, configName, s, exts, cfg.Extensions, excludeFieldDoc) } // Resolve fetches each extension's metadata and walks dependencies // depth-first, so every extension comes after the extensions it depends on. // Each URL appears once. func (g *Generator) Resolve(urls []string) ([]*Extension, error) { var order []*Extension seen := map[string]bool{} visiting := map[string]bool{} declared := map[string]bool{} for _, u := range urls { declared[u] = true } var visit func(u string) error visit = func(u string) error { if seen[u] { return nil } if visiting[u] { return fmt.Errorf("dependency cycle at %s", u) } visiting[u] = true var meta struct { Name map[string]string `json:"name"` Schemas []string `json:"schemas"` Dependencies []string `json:"dependencies"` } body, err := g.fetch(u) if err != nil { return err } if err := json.Unmarshal(body, &meta); err != nil { return fmt.Errorf("%s: %w", u, err) } e := &Extension{ URL: u, Name: meta.Name["en"], Schemas: meta.Schemas, Dependencies: meta.Dependencies, Declared: declared[u], } if e.Name == "" { e.Name = u } for i, d := range meta.Dependencies { // Dependencies are usually absolute URLs; relative ones are // resolved against the extension that declares them. if d, err = resolveRef(u, d); err != nil { return err } e.Dependencies[i] = d if err := visit(d); err != nil { return fmt.Errorf("%s -> %w", e.Name, err) } } visiting[u] = false seen[u] = true order = append(order, e) return nil } for _, u := range urls { if err := visit(u); err != nil { return nil, err } } g.warnDuplicateRepos(order) return order, nil } // Merge fetches the core schemas and applies each extension's patches in // order using JSON Merge Patch, as the OCDS tooling does. func (g *Generator) Merge(cfg *Config, exts []*Extension) (*Schemas, error) { release, err := g.fetchObj(cfg.ReleaseSchema) if err != nil { return nil, err } pkg, err := g.fetchObj(cfg.ReleasePackageSchema) if err != nil { return nil, err } for _, e := range exts { for _, s := range e.Schemas { var target **Obj switch path.Base(s) { case "release-schema.json": target = &release case "release-package-schema.json": target = &pkg default: g.notef("%s: skipping %s (not generated)", e.Name, s) continue } schemaURL, err := resolveRef(e.URL, s) if err != nil { return nil, err } patch, err := g.fetchObj(schemaURL) if err != nil { return nil, fmt.Errorf("%s: %w", e.Name, err) } g.warnTypeConflicts(e.Name, *target, patch) *target = mergePatch(*target, patch).(*Obj) } } return &Schemas{Release: release, Package: pkg}, nil } // warnDuplicateRepos flags the same extension pulled in at two different // versions (e.g. .../master/extension.json and .../v1.1.5/extension.json). func (g *Generator) warnDuplicateRepos(exts []*Extension) { byRepo := map[string][]string{} for _, e := range exts { u, err := url.Parse(e.URL) if err != nil { continue } parts := strings.Split(strings.Trim(u.Path, "/"), "/") if len(parts) >= 2 { repo := strings.ToLower(parts[0] + "/" + parts[1]) byRepo[repo] = append(byRepo[repo], e.URL) } } repos := make([]string, 0, len(byRepo)) for r := range byRepo { repos = append(repos, r) } sort.Strings(repos) for _, r := range repos { if len(byRepo[r]) > 1 { g.warnf("%s is included more than once: %s", r, strings.Join(byRepo[r], ", ")) } } } // warnTypeConflicts reports when a patch changes the JSON type of an existing // property, which usually means two extensions disagree. func (g *Generator) warnTypeConflicts(ext string, target, patch *Obj) { check := func(where string, tProps, pProps *Obj) { if tProps == nil || pProps == nil { return } for _, k := range pProps.Keys { pp, _ := pProps.M[k].(*Obj) tp, _ := tProps.M[k].(*Obj) if pp == nil || tp == nil { continue } if a, b := typeKey(tp), typeKey(pp); a != "" && b != "" && a != b { g.warnf("%s changes %s.%s from %s to %s", ext, where, k, a, b) } } } check("Release", target.Obj("properties"), patch.Obj("properties")) pDefs, tDefs := patch.Obj("definitions"), target.Obj("definitions") if pDefs == nil || tDefs == nil { return } for _, name := range pDefs.Keys { if pd, td := pDefs.Obj(name), tDefs.Obj(name); pd != nil && td != nil { check(name, td.Obj("properties"), pd.Obj("properties")) } } } func typeKey(s *Obj) string { if ref := s.Str("$ref"); ref != "" { return "$ref:" + ref } ts := typeSet(s) sort.Strings(ts) return strings.Join(ts, "|") } func resolveRef(base, ref string) (string, error) { if !strings.Contains(base, "://") { // Local path: URL resolution would turn "a/b.json" + "../c.json" // into "/c.json", so use file path rules instead. if strings.Contains(ref, "://") || filepath.IsAbs(ref) { return ref, nil } return filepath.Join(filepath.Dir(base), ref), nil } b, err := url.Parse(base) if err != nil { return "", err } r, err := url.Parse(ref) if err != nil { return "", err } return b.ResolveReference(r).String(), nil } func (g *Generator) fetch(u string) ([]byte, error) { if b, ok := g.cache[u]; ok { return b, nil } var body []byte if strings.HasPrefix(u, "http://") || strings.HasPrefix(u, "https://") { resp, err := g.client.Get(u) if err != nil { return nil, err } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { return nil, fmt.Errorf("GET %s: %s", u, resp.Status) } if body, err = io.ReadAll(resp.Body); err != nil { return nil, err } } else { // Local paths are allowed, e.g. for a vendored or private extension. var err error if body, err = os.ReadFile(strings.TrimPrefix(u, "file://")); err != nil { return nil, err } } g.cache[u] = body return body, nil } func (g *Generator) fetchObj(u string) (*Obj, error) { body, err := g.fetch(u) if err != nil { return nil, err } v, err := decodeOrdered(bytes.NewReader(body)) if err != nil { return nil, fmt.Errorf("%s: %w", u, err) } o, ok := v.(*Obj) if !ok { return nil, fmt.Errorf("%s: not a JSON object", u) } return o, nil }