bkl/parser.go

// Package bkl implements a layered configuration language parser.
//
//   - Language & tool documentation: https://bkl.gopatchy.io/
//   - Go library source: https://github.com/gopatchy/bkl
//   - Go library documentation: https://pkg.go.dev/github.com/gopatchy/bkl
package bkl

import (
	"errors"
	"fmt"
	"io"
	"io/fs"
	"os"
	"path/filepath"
	"strings"
)

// Debug controls debug log output to stderr for all BKL operations.
var Debug = os.Getenv("BKL_DEBUG") != ""

// A BKL reads input documents, merges layers, and generates outputs.
//
// # Terminology
//   - Each BKL can read multiple files
//   - Each file represents a single layer
//   - Each file contains one or more documents
//   - Each document generates one or more outputs
//
// # Directive Evaluation Order
//
// Directive evaluation order can matter, e.g. if you $merge a subtree that
// contains an $output directive.
//
// Phase 1
//   - $parent
//
// Phase 2
//   - $delete
//   - $replace: true
//
// Phase 3
//   - $merge
//   - $replace: map
//   - $replace: string
//
// Phase 4
//   - $repeat: int
//
// Phase 5
//   - $""
//   - $encode
//   - $decode
//   - $env
//   - $repeat
//   - $value
//
// Phase 6
//   - $output
//
// # Document Layer Matching Logic
//
// When applying a new document to internal state, it may be merged into one or
// more existing documents or appended as a new document. To select merge
// targets, BKL considers (in order):
//   - If $match:
//   - $match: null -> append
//   - $match within parent documents -> merge
//   - $match any documents -> merge
//   - No matching documents -> error
//   - If parent documents -> merge into all parents
//   - If no parent documents -> append
type BKL struct {
	docs []*Document
}

func New() (*BKL, error) {
	return &BKL{}, nil
}

// MergeDocument applies the supplied Document to the [BKL]'s current
// internal document state using bkl's merge semantics. If expand is true,
// documents without $match will append; otherwise this is an error.
func (b *BKL) mergeDocument(patch *Document) error {
	matched, err := b.mergePatchMatch(patch)
	if err != nil {
		return err
	}

	if matched {
		return nil
	}

	for _, doc := range b.parents(patch) {
		matched = true

		err = mergeDocs(doc, patch)
		if err != nil {
			return err
		}
	}

	if !matched {
		b.docs = append(b.docs, patch)
	}

	return nil
}

func (b *BKL) parents(patch *Document) []*Document {
	ret := []*Document{}

	parents := patch.allParents()

	for _, doc := range b.docs {
		if _, found := parents[doc.ID]; found {
			ret = append(ret, doc)
		}
	}

	return ret
}

// mergePatchMatch attempts to apply the supplied patch to one or more
// documents specified by $match. It returns success and error separately;
// (false, nil) means no $match directive. Zero matches is an error.
func (b *BKL) mergePatchMatch(patch *Document) (bool, error) {
	found, m := patch.popMapValue("$match")
	if !found {
		return false, nil
	}

	if m == nil {
		// Explicit append
		doc := newDocument(fmt.Sprintf("%s|matchnull", patch.ID))
		b.docs = append(b.docs, doc)
		return true, mergeDocs(doc, patch)
	}

	docs := b.findMatches(patch, m)
	if len(docs) == 0 {
		return true, fmt.Errorf("%#v: %w", m, ErrNoMatchFound)
	}

	for _, doc := range docs {
		err := mergeDocs(doc, patch)
		if err != nil {
			return true, err
		}
	}

	return true, nil
}

func (b *BKL) findMatches(doc *Document, pat any) []*Document {
	ret := []*Document{}

	// Try parents, then all docs
	for _, ds := range [][]*Document{b.parents(doc), b.docs} {
		for _, d := range ds {
			if matchDoc(d, pat) {
				ret = append(ret, d)
			}
		}

		if len(ret) > 0 {
			return ret
		}
	}

	return nil
}

// MergeFile parses the file at path and merges its contents into the
// [BKL]'s document state using bkl's merge semantics.
func (b *BKL) mergeFile(fsys *fileSystem, path string) error {
	f, err := b.loadFile(fsys, path, nil)
	if err != nil {
		return err
	}

	return b.mergeFileObj(f)
}

// MergeFileLayers determines relevant layers from the supplied path and merges
// them in order.
func (b *BKL) MergeFileLayers(fsys fs.FS, path string) error {
	fileSystem := newFS(fsys)
	files, err := b.loadFileAndParents(fileSystem, path, nil)
	if err != nil {
		return err
	}

	for _, f := range files {
		err := b.mergeFileObj(f)
		if err != nil {
			return err
		}
	}

	return nil
}

// mergeFile applies an already-parsed file object into the [BKL]'s
// document state.
func (b *BKL) mergeFileObj(f *file) error {
	debugLog("[%s] merging", f)

	for _, doc := range f.docs {
		debugLog("[%s] merging", doc)

		err := b.mergeDocument(doc)
		if err != nil {
			return fmt.Errorf("[%s:%s]: %w", f, doc, err)
		}
	}

	return nil
}

// Documents returns the parsed, merged (but not processed) trees for all
// documents.
func (b *BKL) Documents() []*Document {
	return b.docs
}

// outputDocument returns the output objects generated by the specified
// document.
func (b *BKL) outputDocument(doc *Document, env map[string]string) ([]any, error) {
	docs, err := doc.Process(b.docs, env)
	if err != nil {
		return nil, err
	}

	outs := []any{}

	for _, d := range docs {
		obj, out, err := findOutputs(d.Data)
		if err != nil {
			return nil, err
		}

		if len(out) == 0 {
			outs = append(outs, obj)
		} else {
			outs = append(outs, out...)
		}
	}

	return filterList(outs, func(v any) ([]any, error) {
		v2, include, err := filterOutput(v)
		if err != nil {
			return nil, err
		}

		if !include {
			return nil, nil
		}

		err = validate(v2)
		if err != nil {
			return nil, err
		}

		return []any{finalizeOutput(v2)}, nil
	})
}

// OutputDocuments returns the output objects generated by all documents.
func (b *BKL) outputDocuments(env map[string]string) ([]any, error) {
	ret := []any{}

	for _, doc := range b.docs {
		outs, err := b.outputDocument(doc, env)
		if err != nil {
			return nil, err
		}

		ret = append(ret, outs...)
	}

	return ret, nil
}

// Output returns all documents encoded in the specified format and merged into
// a stream.
func (b *BKL) output(format string, env map[string]string) ([]byte, error) {
	outs, err := b.outputDocuments(env)
	if err != nil {
		return nil, err
	}

	f, err := getFormat(format)
	if err != nil {
		return nil, err
	}

	return f.MarshalStream(outs)
}

// OutputToFile encodes all documents in the specified format and writes them
// to the specified output path.
//
// If format is "", it is inferred from path's file extension.
func (b *BKL) OutputToFile(path, format string, env map[string]string) error {
	if format == "" {
		format = b.Ext(path)
	}

	fh, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o644)
	if err != nil {
		return errors.Join(fmt.Errorf("%s: %w", path, ErrOutputFile), err)
	}

	defer fh.Close()

	err = b.outputToWriter(fh, format, env)
	if err != nil {
		return fmt.Errorf("%s: %w", path, err)
	}

	return nil
}

// OutputToWriter encodes all documents in the specified format and writes them
// to the specified [io.Writer].
//
// If format is "", it defaults to "json-pretty".
func (b *BKL) outputToWriter(fh io.Writer, format string, env map[string]string) error {
	if format == "" {
		format = "json-pretty"
	}

	out, err := b.output(format, env)
	if err != nil {
		return err
	}

	_, err = fh.Write(out)
	if err != nil {
		return errors.Join(ErrOutputFile, err)
	}

	return nil
}

func (b *BKL) makePathsAbsolute(paths []string, workingDir string) ([]string, error) {
	result := make([]string, len(paths))
	for i, path := range paths {
		if filepath.IsAbs(path) {
			result[i] = path
		} else {
			result[i] = filepath.Join(workingDir, path)
		}
	}
	return result, nil
}

func (b *BKL) rebasePathsToRoot(absPaths []string, rootPath string, workingDir string) ([]string, error) {
	absRootPath := rootPath
	if !filepath.IsAbs(rootPath) {
		absRootPath = filepath.Join(workingDir, rootPath)
	}

	result := make([]string, len(absPaths))
	for i, path := range absPaths {
		relPath, err := filepath.Rel(absRootPath, path)
		if err != nil {
			return nil, fmt.Errorf("file %s outside root path: %w", path, err)
		}

		if strings.HasPrefix(relPath, "..") {
			return nil, fmt.Errorf("file %s outside root path", path)
		}

		result[i] = "/" + relPath
	}

	return result, nil
}

func (b *BKL) preparePathsForParser(paths []string, rootPath string, workingDir string) ([]string, error) {
	absPaths, err := b.makePathsAbsolute(paths, workingDir)
	if err != nil {
		return nil, err
	}

	return b.rebasePathsToRoot(absPaths, rootPath, workingDir)
}

// PreparePathsFromCwd prepares file paths relative to the current working directory
// and rebases them to the given root path.
func (b *BKL) PreparePathsFromCwd(paths []string, rootPath string) ([]string, error) {
	wd, err := os.Getwd()
	if err != nil {
		return nil, err
	}

	return b.preparePathsForParser(paths, rootPath, wd)
}

// GetOSEnv returns the current OS environment as a map.
func (b *BKL) GetOSEnv() map[string]string {
	env := make(map[string]string)
	for _, e := range os.Environ() {
		parts := strings.SplitN(e, "=", 2)
		if len(parts) == 2 {
			env[parts[0]] = parts[1]
		}
	}
	return env
}

// Ext returns the file extension without the leading dot.
func (b *BKL) Ext(path string) string {
	return strings.TrimPrefix(filepath.Ext(path), ".")
}

// FormatOutput marshals the given data to the specified format.
// Returns the marshaled bytes or an error if the format is unknown or marshaling fails.
func (b *BKL) FormatOutput(data any, format string) ([]byte, error) {
	f, found := formatByExtension[format]
	if !found {
		return nil, fmt.Errorf("%s: %w", format, ErrUnknownFormat)
	}

	// Always wrap in a slice for MarshalStream - it expects a stream of documents
	return f.MarshalStream([]any{data})
}

func (b *BKL) Evaluate(fsys fs.FS, files []string, skipParent bool, format string, rootPath string, workingDir string, env map[string]string) ([]byte, error) {
	evalFiles, err := b.preparePathsForParser(files, rootPath, workingDir)
	if err != nil {
		return nil, err
	}

	// Set environment variables for evaluation

	for _, path := range evalFiles {
		realPath, fileFormat, err := b.FileMatch(fsys, path)
		if err != nil {
			return nil, fmt.Errorf("file %s: %w", path, err)
		}

		if format == "" {
			format = fileFormat
		}

		if skipParent {
			fileSystem := newFS(fsys)
			err = b.mergeFile(fileSystem, realPath)
		} else {
			err = b.MergeFileLayers(fsys, realPath)
		}

		if err != nil {
			return nil, fmt.Errorf("merging %s: %w", path, err)
		}
	}

	return b.output(format, env)
}

// EvaluateToData is like Evaluate but returns the raw data instead of marshaled output
func (b *BKL) EvaluateToData(fsys fs.FS, files []string, skipParent bool, format string, rootPath string, workingDir string, env map[string]string) (any, error) {
	evalFiles, err := b.preparePathsForParser(files, rootPath, workingDir)
	if err != nil {
		return nil, err
	}

	for _, path := range evalFiles {
		realPath, fileFormat, err := b.FileMatch(fsys, path)
		if err != nil {
			return nil, fmt.Errorf("file %s: %w", path, err)
		}

		if format == "" {
			format = fileFormat
		}

		if skipParent {
			fileSystem := newFS(fsys)
			err = b.mergeFile(fileSystem, realPath)
		} else {
			err = b.MergeFileLayers(fsys, realPath)
		}

		if err != nil {
			return nil, fmt.Errorf("merging %s: %w", path, err)
		}
	}

	// Get the raw output data
	outs, err := b.outputDocuments(env)
	if err != nil {
		return nil, err
	}

	// Merge all outputs into a single value
	if len(outs) == 0 {
		return nil, nil
	} else if len(outs) == 1 {
		return outs[0], nil
	} else {
		// Multiple documents - return as list
		return outs, nil
	}
}

// FileMatch attempts to find a file with the same base name as path, but
// possibly with a different supported extension. It is intended to support
// "virtual" filenames that auto-convert from the format of the underlying
// real file.
//
// Returns the real filename and the requested output format, or
// ("", "", error).
func (b *BKL) FileMatch(fsys fs.FS, path string) (string, string, error) {
	format := b.Ext(path)
	if _, found := formatByExtension[format]; !found {
		return "", "", fmt.Errorf("%s: %w", format, ErrUnknownFormat)
	}

	withoutExt := strings.TrimSuffix(path, "."+format)

	if filepath.Base(withoutExt) == "-" {
		return path, format, nil
	}

	fileSystem := newFS(fsys)
	realPath := fileSystem.findFile(withoutExt)

	if realPath == "" {
		return "", "", fmt.Errorf("%s.*: %w", withoutExt, ErrMissingFile)
	}

	return realPath, format, nil
}