kittenipc/kitcom/internal/tsgo/execute/build/orchestrator.go

package build

import (
	"io"
	"strings"
	"sync/atomic"
	"time"

	"efprojects.com/kitten-ipc/kitcom/internal/tsgo/ast"
	"efprojects.com/kitten-ipc/kitcom/internal/tsgo/collections"
	"efprojects.com/kitten-ipc/kitcom/internal/tsgo/compiler"
	"efprojects.com/kitten-ipc/kitcom/internal/tsgo/core"
	"efprojects.com/kitten-ipc/kitcom/internal/tsgo/diagnostics"
	"efprojects.com/kitten-ipc/kitcom/internal/tsgo/execute/tsc"
	"efprojects.com/kitten-ipc/kitcom/internal/tsgo/tsoptions"
	"efprojects.com/kitten-ipc/kitcom/internal/tsgo/tspath"
	"efprojects.com/kitten-ipc/kitcom/internal/tsgo/vfs/cachedvfs"
)

type Options struct {
	Sys     tsc.System
	Command *tsoptions.ParsedBuildCommandLine
	Testing tsc.CommandLineTesting
}

type orchestratorResult struct {
	result        tsc.CommandLineResult
	errors        []*ast.Diagnostic
	statistics    tsc.Statistics
	filesToDelete []string
}

func (b *orchestratorResult) report(o *Orchestrator) {
	if o.opts.Command.CompilerOptions.Watch.IsTrue() {
		o.watchStatusReporter(ast.NewCompilerDiagnostic(core.IfElse(len(b.errors) == 1, diagnostics.Found_1_error_Watching_for_file_changes, diagnostics.Found_0_errors_Watching_for_file_changes), len(b.errors)))
	} else {
		o.errorSummaryReporter(b.errors)
	}
	if b.filesToDelete != nil {
		o.createBuilderStatusReporter(nil)(
			ast.NewCompilerDiagnostic(
				diagnostics.A_non_dry_build_would_delete_the_following_files_Colon_0,
				strings.Join(core.Map(b.filesToDelete, func(f string) string {
					return "\r\n * " + f
				}), ""),
			))
	}
	if !o.opts.Command.CompilerOptions.Diagnostics.IsTrue() && !o.opts.Command.CompilerOptions.ExtendedDiagnostics.IsTrue() {
		return
	}
	b.statistics.SetTotalTime(o.opts.Sys.SinceStart())
	b.statistics.Report(o.opts.Sys.Writer(), o.opts.Testing)
}

type Orchestrator struct {
	opts                Options
	comparePathsOptions tspath.ComparePathsOptions
	host                *host

	// order generation result
	tasks  *collections.SyncMap[tspath.Path, *buildTask]
	order  []string
	errors []*ast.Diagnostic

	errorSummaryReporter tsc.DiagnosticsReporter
	watchStatusReporter  tsc.DiagnosticReporter
}

var _ tsc.Watcher = (*Orchestrator)(nil)

func (o *Orchestrator) relativeFileName(fileName string) string {
	return tspath.ConvertToRelativePath(fileName, o.comparePathsOptions)
}

func (o *Orchestrator) toPath(fileName string) tspath.Path {
	return tspath.ToPath(fileName, o.comparePathsOptions.CurrentDirectory, o.comparePathsOptions.UseCaseSensitiveFileNames)
}

func (o *Orchestrator) Order() []string {
	return o.order
}

func (o *Orchestrator) Upstream(configName string) []string {
	path := o.toPath(configName)
	task := o.getTask(path)
	return core.Map(task.upStream, func(t *upstreamTask) string {
		return t.task.config
	})
}

func (o *Orchestrator) Downstream(configName string) []string {
	path := o.toPath(configName)
	task := o.getTask(path)
	return core.Map(task.downStream, func(t *buildTask) string {
		return t.config
	})
}

func (o *Orchestrator) getTask(path tspath.Path) *buildTask {
	task, ok := o.tasks.Load(path)
	if !ok {
		panic("No build task found for " + path)
	}
	return task
}

func (o *Orchestrator) createBuildTasks(oldTasks *collections.SyncMap[tspath.Path, *buildTask], configs []string, wg core.WorkGroup) {
	for _, config := range configs {
		wg.Queue(func() {
			path := o.toPath(config)
			var task *buildTask
			var buildInfo *buildInfoEntry
			if oldTasks != nil {
				if existing, ok := oldTasks.Load(path); ok {
					if !existing.dirty {
						// Reuse existing task if config is same
						task = existing
					} else {
						buildInfo = existing.buildInfoEntry
					}
				}
			}
			if task == nil {
				task = &buildTask{config: config, isInitialCycle: oldTasks == nil}
				task.pending.Store(true)
				task.buildInfoEntry = buildInfo
			}
			if _, loaded := o.tasks.LoadOrStore(path, task); loaded {
				return
			}
			task.resolved = o.host.GetResolvedProjectReference(config, path)
			task.upStream = nil
			if task.resolved != nil {
				o.createBuildTasks(oldTasks, task.resolved.ResolvedProjectReferencePaths(), wg)
			}
		})
	}
}

func (o *Orchestrator) setupBuildTask(
	configName string,
	downStream *buildTask,
	inCircularContext bool,
	completed *collections.Set[tspath.Path],
	analyzing *collections.Set[tspath.Path],
	circularityStack []string,
) *buildTask {
	path := o.toPath(configName)
	task := o.getTask(path)
	if !completed.Has(path) {
		if analyzing.Has(path) {
			if !inCircularContext {
				o.errors = append(o.errors, ast.NewCompilerDiagnostic(
					diagnostics.Project_references_may_not_form_a_circular_graph_Cycle_detected_Colon_0,
					strings.Join(circularityStack, "\n"),
				))
			}
			return nil
		}
		analyzing.Add(path)
		circularityStack = append(circularityStack, configName)
		if task.resolved != nil {
			for index, subReference := range task.resolved.ResolvedProjectReferencePaths() {
				upstream := o.setupBuildTask(subReference, task, inCircularContext || task.resolved.ProjectReferences()[index].Circular, completed, analyzing, circularityStack)
				if upstream != nil {
					task.upStream = append(task.upStream, &upstreamTask{task: upstream, refIndex: index})
				}
			}
		}
		circularityStack = circularityStack[:len(circularityStack)-1]
		completed.Add(path)
		task.reportDone = make(chan struct{})
		prev := core.LastOrNil(o.order)
		if prev != "" {
			task.prevReporter = o.getTask(o.toPath(prev))
		}
		task.done = make(chan struct{})
		o.order = append(o.order, configName)
	}
	if o.opts.Command.CompilerOptions.Watch.IsTrue() && downStream != nil {
		task.downStream = append(task.downStream, downStream)
	}
	return task
}

func (o *Orchestrator) GenerateGraphReusingOldTasks() {
	tasks := o.tasks
	o.tasks = &collections.SyncMap[tspath.Path, *buildTask]{}
	o.order = nil
	o.errors = nil
	o.GenerateGraph(tasks)
}

func (o *Orchestrator) GenerateGraph(oldTasks *collections.SyncMap[tspath.Path, *buildTask]) {
	projects := o.opts.Command.ResolvedProjectPaths()
	// Parse all config files in parallel
	wg := core.NewWorkGroup(o.opts.Command.CompilerOptions.SingleThreaded.IsTrue())
	o.createBuildTasks(oldTasks, projects, wg)
	wg.RunAndWait()

	// Generate the graph
	completed := collections.Set[tspath.Path]{}
	analyzing := collections.Set[tspath.Path]{}
	circularityStack := []string{}
	for _, project := range projects {
		o.setupBuildTask(project, nil, false, &completed, &analyzing, circularityStack)
	}
}

func (o *Orchestrator) Start() tsc.CommandLineResult {
	if o.opts.Command.CompilerOptions.Watch.IsTrue() {
		o.watchStatusReporter(ast.NewCompilerDiagnostic(diagnostics.Starting_compilation_in_watch_mode))
	}
	o.GenerateGraph(nil)
	result := o.buildOrClean()
	if o.opts.Command.CompilerOptions.Watch.IsTrue() {
		o.Watch()
		result.Watcher = o
	}
	return result
}

func (o *Orchestrator) Watch() {
	o.updateWatch()
	o.resetCaches()

	// Start watching for file changes
	if o.opts.Testing == nil {
		watchInterval := o.opts.Command.WatchOptions.WatchInterval()
		for {
			// Testing mode: run a single cycle and exit
			time.Sleep(watchInterval)
			o.DoCycle()
		}
	}
}

func (o *Orchestrator) updateWatch() {
	oldCache := o.host.mTimes
	o.host.mTimes = &collections.SyncMap[tspath.Path, time.Time]{}
	wg := core.NewWorkGroup(o.opts.Command.CompilerOptions.SingleThreaded.IsTrue())
	o.tasks.Range(func(path tspath.Path, task *buildTask) bool {
		wg.Queue(func() {
			task.updateWatch(o, oldCache)
		})
		return true
	})
	wg.RunAndWait()
}

func (o *Orchestrator) resetCaches() {
	// Clean out all the caches
	cachesVfs := o.host.host.FS().(*cachedvfs.FS)
	cachesVfs.ClearCache()
	o.host.extendedConfigCache = tsc.ExtendedConfigCache{}
	o.host.sourceFiles.reset()
	o.host.configTimes = collections.SyncMap[tspath.Path, time.Duration]{}
}

func (o *Orchestrator) DoCycle() {
	var needsConfigUpdate atomic.Bool
	var needsUpdate atomic.Bool
	mTimes := o.host.mTimes.Clone()
	wg := core.NewWorkGroup(o.opts.Command.CompilerOptions.SingleThreaded.IsTrue())
	o.tasks.Range(func(path tspath.Path, task *buildTask) bool {
		wg.Queue(func() {
			if updateKind := task.hasUpdate(o, path); updateKind != updateKindNone {
				needsUpdate.Store(true)
				if updateKind == updateKindConfig {
					needsConfigUpdate.Store(true)
				}
			}
		})
		// Watch for file changes
		return true
	})
	wg.RunAndWait()

	if !needsUpdate.Load() {
		o.host.mTimes = mTimes
		o.resetCaches()
		return
	}

	o.watchStatusReporter(ast.NewCompilerDiagnostic(diagnostics.File_change_detected_Starting_incremental_compilation))
	if needsConfigUpdate.Load() {
		// Generate new tasks
		o.GenerateGraphReusingOldTasks()
	}

	o.buildOrClean()
	o.updateWatch()
	o.resetCaches()
}

func (o *Orchestrator) buildOrClean() tsc.CommandLineResult {
	if !o.opts.Command.BuildOptions.Clean.IsTrue() && o.opts.Command.BuildOptions.Verbose.IsTrue() {
		o.createBuilderStatusReporter(nil)(ast.NewCompilerDiagnostic(
			diagnostics.Projects_in_this_build_Colon_0,
			strings.Join(core.Map(o.Order(), func(p string) string {
				return "\r\n    * " + o.relativeFileName(p)
			}), ""),
		))
	}
	var buildResult orchestratorResult
	if len(o.errors) == 0 {
		buildResult.statistics.Projects = len(o.Order())
		if o.opts.Command.CompilerOptions.SingleThreaded.IsTrue() {
			o.singleThreadedBuildOrClean(&buildResult)
		} else {
			o.multiThreadedBuildOrClean(&buildResult)
		}
	} else {
		// Circularity errors prevent any project from being built
		buildResult.result.Status = tsc.ExitStatusProjectReferenceCycle_OutputsSkipped
		reportDiagnostic := o.createDiagnosticReporter(nil)
		for _, err := range o.errors {
			reportDiagnostic(err)
		}
		buildResult.errors = o.errors
	}
	buildResult.report(o)
	return buildResult.result
}

func (o *Orchestrator) singleThreadedBuildOrClean(buildResult *orchestratorResult) {
	// Go in the order since only one project can be built at a time so that random order isnt picked by work group creating deadlock
	for _, config := range o.Order() {
		path := o.toPath(config)
		task := o.getTask(path)
		o.buildOrCleanProject(task, path, buildResult)
	}
}

func (o *Orchestrator) multiThreadedBuildOrClean(buildResult *orchestratorResult) {
	// Spin off the threads with waiting on upstream to build before actual project build
	wg := core.NewWorkGroup(false)
	o.tasks.Range(func(path tspath.Path, task *buildTask) bool {
		wg.Queue(func() {
			o.buildOrCleanProject(task, path, buildResult)
		})
		return true
	})
	wg.RunAndWait()
}

func (o *Orchestrator) buildOrCleanProject(task *buildTask, path tspath.Path, buildResult *orchestratorResult) {
	task.result = &taskResult{}
	task.result.reportStatus = o.createBuilderStatusReporter(task)
	task.result.diagnosticReporter = o.createDiagnosticReporter(task)
	if !o.opts.Command.BuildOptions.Clean.IsTrue() {
		task.buildProject(o, path)
	} else {
		task.cleanProject(o, path)
	}
	task.report(o, path, buildResult)
}

func (o *Orchestrator) getWriter(task *buildTask) io.Writer {
	if task == nil {
		return o.opts.Sys.Writer()
	}
	return &task.result.builder
}

func (o *Orchestrator) createBuilderStatusReporter(task *buildTask) tsc.DiagnosticReporter {
	return tsc.CreateBuilderStatusReporter(o.opts.Sys, o.getWriter(task), o.opts.Command.CompilerOptions, o.opts.Testing)
}

func (o *Orchestrator) createDiagnosticReporter(task *buildTask) tsc.DiagnosticReporter {
	return tsc.CreateDiagnosticReporter(o.opts.Sys, o.getWriter(task), o.opts.Command.CompilerOptions)
}

func NewOrchestrator(opts Options) *Orchestrator {
	orchestrator := &Orchestrator{
		opts: opts,
		comparePathsOptions: tspath.ComparePathsOptions{
			CurrentDirectory:          opts.Sys.GetCurrentDirectory(),
			UseCaseSensitiveFileNames: opts.Sys.FS().UseCaseSensitiveFileNames(),
		},
		tasks: &collections.SyncMap[tspath.Path, *buildTask]{},
	}
	orchestrator.host = &host{
		orchestrator: orchestrator,
		host: compiler.NewCachedFSCompilerHost(
			orchestrator.opts.Sys.GetCurrentDirectory(),
			orchestrator.opts.Sys.FS(),
			orchestrator.opts.Sys.DefaultLibraryPath(),
			nil,
			nil,
		),
		mTimes: &collections.SyncMap[tspath.Path, time.Time]{},
	}
	if opts.Command.CompilerOptions.Watch.IsTrue() {
		orchestrator.watchStatusReporter = tsc.CreateWatchStatusReporter(opts.Sys, opts.Command.CompilerOptions, opts.Testing)
	} else {
		orchestrator.errorSummaryReporter = tsc.CreateReportErrorSummary(opts.Sys, opts.Command.CompilerOptions)
	}
	return orchestrator
}