diff options
Diffstat (limited to 'src')
47 files changed, 3452 insertions, 0 deletions
diff --git a/src/BuildPipeline.cs b/src/BuildPipeline.cs new file mode 100644 index 0000000..2435566 --- /dev/null +++ b/src/BuildPipeline.cs @@ -0,0 +1,305 @@ +using System; +using System.IO; +using System.Linq; +using System.Threading.Tasks; +using System.Collections.Generic; + +using Serilog; +using Serilog.Core; + +using VNLib.Tools.Build.Executor.Model; +using VNLib.Tools.Build.Executor.Modules; +using VNLib.Tools.Build.Executor.Extensions; +using VNLib.Tools.Build.Executor.Constants; + +namespace VNLib.Tools.Build.Executor +{ + + public sealed class BuildPipeline(Logger Log) : IDisposable + { + private readonly List<ModuleBase> _allModules = new(); + private readonly List<ModuleBase> _selected = new(); + private readonly LinkedList<ModuleBase> _outdatedModules = new(); + private readonly LinkedList<IProject> _modifiedProjects = new(); + private readonly TaskfileVars _taskVars = new(); + + /// <summary> + /// Loads a modules within the working directory + /// </summary> + /// <returns>A task that completes when all modules and child projects are loaded</returns> + public async Task LoadAsync(BuildConfig config, string[] only, string[] exclude, IFeedManager[] feeds) + { + //Init task variables + SetTaskVariables(config.Index, feeds); + + //Capture all modules within pwd + Log.Information("Discovering modules in {pwd}", config.Index.BaseDir.FullName); + + //Search for .git repos + DirectoryInfo[] moduleDirs = config.Index.BaseDir.EnumerateDirectories(".git", SearchOption.AllDirectories) + .Select(static s => s.Parent!) + .ToArray(); + + //Add modules + foreach(DirectoryInfo dir in moduleDirs) + { + _allModules.Add(new GitCodeModule(config, dir)); + } + + Log.Information("Found {c} modules, loading modules...", moduleDirs.Length); + + //Load all modules async and give them each a copy of our local task variables + await _allModules.RunAllAsync(p => p.LoadAsync(_taskVars.Clone())); + + //Only include desired modules + if (only.Length > 0) + { + Log.Information("Only including modules {mods}", only); + + ModuleBase[] onlyMods = _allModules.Where(m => only.Contains(m.ModuleName, StringComparer.OrdinalIgnoreCase)).ToArray(); + _selected.AddRange(onlyMods); + } + //Exclude given modules + else if(exclude.Length > 0) + { + Log.Information("Excluding modules {mods}", exclude); + + ModuleBase[] excludeMods = _allModules.Where(m => exclude.Contains(m.ModuleName, StringComparer.OrdinalIgnoreCase)).ToArray(); + _selected.AddRange(_allModules.Except(excludeMods)); + } + else + { + //Just all all modules to the list + _selected.AddRange(_allModules); + } + + Log.Information("The following modules will be processed\n{mods}", _selected.Select(m => m.ModuleName)); + } + + private void SetTaskVariables(IDirectoryIndex dirIndex, IFeedManager[] feeds) + { + //Configure variables + _taskVars.Set("BUILD_DIR", dirIndex.BuildDir.FullName); + _taskVars.Set("SCRATCH_DIR", dirIndex.ScratchDir.FullName); + _taskVars.Set("UNIX_MS", DateTimeOffset.UtcNow.ToUnixTimeMilliseconds().ToString()); + _taskVars.Set("DATE", DateTimeOffset.Now.ToString("d")); + + //Add all feed manager to task variables + Array.ForEach(feeds, f => f.AddVariables(_taskVars)); + } + + /// <summary> + /// Synchronizes all modules with their respective remote repositories + /// </summary> + /// <returns></returns> + public async Task DoStepUpdateSource() + { + //Clear outdated list before syncing sources + _outdatedModules.Clear(); + _modifiedProjects.Clear(); + + //Must sync source serially to prevent git errors + foreach (ModuleBase module in _selected) + { + //Sync source + await module.DoStepSyncSource(); + } + } + + /// <summary> + /// Prepares the build pipeline for building, finds for changes and determines dependencies + /// then prepares modules for building + /// </summary> + /// <returns></returns> + public async Task<bool> CheckForChangesAsync() + { + //Clear outdated list before syncing sources + _outdatedModules.Clear(); + _modifiedProjects.Clear(); + + //Conccurrently search for changes in all modules + await _selected.RunAllAsync(async m => + { + if (await m.CheckForChangesAsync()) + { + _outdatedModules.AddLast(m); + Log.Information("Module {m} MODIFIED. Queued for rebuild", m.ModuleName); + } + }); + + //if one or more modules have been modified, we need to determine dependencies + if (_outdatedModules.Count > 0) + { + //Get the initial list of projects that will be rebuilt + string[] outDatedProjects = _outdatedModules.SelectMany(static m => m.Projects.Where(static p => !p.UpToDate).Select(static p => p.ProjectFile.Name)).ToArray(); + + do + { + + /* + * Select only up-to-date modules + * that have external project references to outdated + * projects + */ + ModuleBase[] dependants = _selected.Where(m => !_outdatedModules.Contains(m)) + .Where( + m => m.GetExternalDependencies() + .Where(externProj => outDatedProjects.Contains(externProj)) + .Any()) + .ToArray(); + + //If there are no more dependants, exit loop + if (dependants.Length == 0) + { + break; + } + + //Add modules to oudated list + for (int i = 0; i < dependants.Length; i++) + { + Log.Information("Module {mod} OUTDATED because it depends on out-of-date modules", dependants[i].ModuleName); + _outdatedModules.AddLast(dependants[i]); + } + + //update outdated projects list to include projects from the newly outdated modules + outDatedProjects = dependants.SelectMany(static p => p.GetExternalDependencies()).ToArray(); + } + while (true); + } + + Log.Information("{c} modules detected source code changes", _outdatedModules.Count); + return _outdatedModules.Count > 0; + } + + public async Task DoStepBuild(bool force) + { + //Rebuild all modules + if (force) + { + //rebuild all selected modules + foreach (ModuleBase mod in _selected) + { + //Run each module independently + await BuildSingleModule(mod, Log); + } + } + else + { + if (_outdatedModules.Count == 0) + { + Log.Information("No modules detected changes"); + } + + //Only rebuild modified modules + foreach (ModuleBase mod in _outdatedModules) + { + //Run each module independently + await BuildSingleModule(mod, Log); + } + } + } + + static async Task BuildSingleModule(IBuildable module, ILogger log) + { + log.Information("Building module {m}", (module as ModuleBase)!.ModuleName); + + try + { + //Build module + await module.DoStepBuild(); + } + catch + { + //failure + await module.DoStepPostBuild(false); + throw; + } + + //Completed successfully, await the result of post-build + await module.DoStepPostBuild(true); + } + + public async Task OnPublishingAsync() + { + /* + * Exec publish step on modules in order incase they + * need to access synchronous resources + */ + + foreach(ModuleBase module in _selected) + { + await module.DoStepPublish(); + } + } + + public async Task PrepareOutputAsync(BuildPublisher publisher) + { + Log.Information("Preparing pipline output"); + + if(publisher.SignEnabled) + { + //Sign all modules synchronously so gpg-agent doesn't get overloaded + foreach (IModuleData module in _selected) + { + //Sign module + await publisher.PrepareModuleOutput(module); + } + } + else + { + await _selected.RunAllAsync(publisher.PrepareModuleOutput); + } + } + + /// <summary> + /// Executes test commands for all loaded modules + /// </summary> + /// <returns></returns> + public async Task ExecuteTestsAsync(bool failOnError) + { + foreach (ModuleBase module in _selected) + { + await module.DoRunTests(failOnError); + } + } + + /// <summary> + /// Performs a manual upload step + /// </summary> + /// <returns></returns> + public async Task ManualUpload(BuildPublisher publisher, IUploadManager uploads) + { + //Upload module output + foreach (IModuleData module in _selected) + { + //Upload module + await publisher.UploadModuleOutput(uploads, module); + } + } + + /// <summary> + /// Cleans all modules and child projects + /// </summary> + /// <returns>A task that resolves when all child projects have been cleaned</returns> + public async Task DoStepCleanAsync() + { + //Clean synchronously + foreach (IArtifact module in _selected) + { + await module.CleanAsync(); + } + } + + public void Dispose() + { + foreach (IArtifact module in _allModules) + { + module.Dispose(); + } + + //Cleanup internals + _outdatedModules.Clear(); + } + + } +}
\ No newline at end of file diff --git a/src/BuildPublisher.cs b/src/BuildPublisher.cs new file mode 100644 index 0000000..a2dc2bf --- /dev/null +++ b/src/BuildPublisher.cs @@ -0,0 +1,430 @@ +using System; +using System.IO; +using System.Linq; +using System.Text; +using System.Text.Json; +using System.Threading.Tasks; +using System.Collections.Generic; + +using LibGit2Sharp; + +using Semver; + +using VNLib.Tools.Build.Executor.Constants; +using VNLib.Tools.Build.Executor.Model; +using VNLib.Tools.Build.Executor.Extensions; +using VNLib.Tools.Build.Executor.Projects; +using static VNLib.Tools.Build.Executor.Constants.Config; + +namespace VNLib.Tools.Build.Executor +{ + + public sealed class BuildPublisher(BuildConfig config, GpgSigner signer) + { + public bool SignEnabled => signer.IsEnabled; + + /// <summary> + /// Prepares the module output and its collection of file details for publishing + /// then runs the upload step + /// </summary> + /// <param name="module"></param> + /// <returns>A task that completes when the module's output has been created</returns> + public async Task PrepareModuleOutput(IModuleData module) + { + //Copy project artifacts to output directory + await CopyProjectOutputToModuleOutputAsync(module); + + //Copy source archive + string? archiveFile = await CopySourceArchiveToOutput(module, module.FileManager); + + Log.Information("Building module {mod} catalog and git history", module.ModuleName); + + //Build module catalog + await BuildModuleCatalogAsync(module, config.SemverStyle, archiveFile); + + Log.Information("Building module {mod} git history", module.ModuleName); + + //Build git history + await BuildModuleGitHistoryAsync(module); + + Log.Information("Building module {mod} version history", module.ModuleName); + + //build version history + await BuildModuleVersionHistory(module); + + Log.Information("Moving module {mod} artifacts to the output", module.ModuleName); + } + + /// <summary> + /// Uploads the modules output to the remote + /// </summary> + /// <param name="module">The module containing the information to upload</param> + /// <returns></returns> + public Task UploadModuleOutput(IUploadManager Uploader, IModuleData module) + { + Log.Information("Uploading module {mod}", module.ModuleName); + + //Upload the entire output directory + return Uploader.UploadDirectoryAsync(module.FileManager.OutputDir); + } + + /* + * Builds the project catalog file and publishes it to the module file manager + */ + private async Task BuildModuleCatalogAsync(IModuleData mod, SemVersionStyles style, string? archiveFile) + { + /* + * Builds the index.json file for the module. It + * contains an array of projects and their metadata + */ + + string moduleVersion = mod.GetModuleCiVersion(config.DefaultCiVersion, style); + + using MemoryStream ms = new(); + + using (Utf8JsonWriter writer = new(ms)) + { + //Open initial object + writer.WriteStartObject(); + + InitModuleFile(writer, mod); + + //Add the archive path if it was created in the module + if (archiveFile != null) + { + writer.WriteStartObject("archive"); + + //Archive path is in the build directory + writer.WriteString("path", config.SourceArchiveName); + + //Get the checksum of the archive + string checksum = await new FileInfo(archiveFile).ComputeFileHashStringAsync(); + writer.WriteString(config.HashFuncName, checksum); + writer.WriteString("sha_file", $"{config.SourceArchiveName}.{config.HashFuncName}"); + + //If signing is enabled, add the signature file (it is constant) + if (SignEnabled) + { + writer.WriteString("signature", $"{config.SourceArchiveName}.sig"); + } + + writer.WriteEndObject(); + } + + //Build project array + writer.WriteStartArray("projects"); + + foreach (IProject project in mod.Projects) + { + //start object for each project + writer.WriteStartObject(); + + //Write the project info + await WriteProjectInfoAsync(mod.FileManager, project, mod.Repository.Head.Tip.Sha, moduleVersion, writer); + + writer.WriteEndObject(); + } + + writer.WriteEndArray(); + + //Close object + writer.WriteEndObject(); + + writer.Flush(); + } + + ms.Seek(0, SeekOrigin.Begin); + + await mod.FileManager.WriteFileAsync(ModuleFileType.Catalog, ms.ToArray()); + } + + private static async Task BuildModuleVersionHistory(IModuleData mod) + { + /* + * Builds the index.json file for the module. It + * contains an array of projects and their metadata + */ + + using MemoryStream ms = new(); + + using (Utf8JsonWriter writer = new(ms)) + { + //Open initial object + writer.WriteStartObject(); + + InitModuleFile(writer, mod); + + //Set the head pointer to the latest commit we build + writer.WriteString("head", mod.Repository.Head.Tip.Sha); + + //Build project array + writer.WriteStartArray("versions"); + + //Write all git hashes from head back to the first commit + foreach(Commit commit in mod.Repository.Commits) + { + writer.WriteStringValue(commit.Sha); + } + + writer.WriteEndArray(); + + //Releases will be an array of objects containing the tag and the hash + writer.WriteStartArray("releases"); + + //Write all git tags + foreach(Tag tag in mod.Repository.Tags.OrderByDescending(static p => p.FriendlyName)) + { + writer.WriteStartObject(); + + writer.WriteString("tag", tag.FriendlyName); + writer.WriteString("hash", tag.Target.Sha); + writer.WriteEndObject(); + } + + writer.WriteEndArray(); + + //Close object + writer.WriteEndObject(); + + writer.Flush(); + } + + ms.Seek(0, SeekOrigin.Begin); + + await mod.FileManager.WriteFileAsync(ModuleFileType.VersionHistory, ms.ToArray()); + } + + + /* + * Builds the project's git history file and publishes it to the module file manager + * + * Also updates the latest hash file + */ + private static async Task BuildModuleGitHistoryAsync(IModuleData mod) + { + using MemoryStream ms = new(); + + using (Utf8JsonWriter writer = new(ms)) + { + //Open initial object + writer.WriteStartObject(); + + InitModuleFile(writer, mod); + + //Write the head commit + writer.WriteStartObject("head"); + writer.WriteString("branch", mod.Repository.Head.FriendlyName); + WriteSingleCommit(writer, mod.Repository.Head.Tip); + writer.WriteEndObject(); + + //Write commit history + WriteCommitHistory(writer, mod.Repository); + + //Close object + writer.WriteEndObject(); + + writer.Flush(); + } + + ms.Seek(0, SeekOrigin.Begin); + + await mod.FileManager.WriteFileAsync(ModuleFileType.GitHistory, ms.ToArray()); + + await mod.FileManager.WriteFileAsync(ModuleFileType.LatestHash, Encoding.UTF8.GetBytes(mod.Repository.Head.Tip.Sha)); + } + + /* + * Captures all of the project artiacts and copies them to the module output directory + */ + private async Task CopyProjectOutputToModuleOutputAsync(IModuleData mod) + { + //Copy build artifacts to module output directory + await mod.Projects.RunAllAsync(project => + { + //Get all output files from the project build, and copy them to the module output directory + return project.GetProjectBuildFiles(config) + .RunAllAsync(artifact => mod.FileManager.CopyArtifactToOutputAsync(project, artifact)); + }); + + /* + * If signing is enabled, we can sign all project files synchronously + */ + if (SignEnabled) + { + Log.Information("GPG Siginig is enabled, signing all artifacts for module {mod}", mod.ModuleName); + + /* + * Get all of the artifacts from the module's projects that match the target output + * file type, and sign them + */ + IEnumerable<FileInfo> artifacts = mod.Projects.SelectMany( + p => mod.FileManager.GetArtifactOutputDir(p) + .EnumerateFiles(config.OutputFileType, SearchOption.TopDirectoryOnly) + ); + + //Sign synchronously + foreach(FileInfo artifact in artifacts) + { + await signer.SignFileAsync(artifact); + } + } + } + + + private static void InitModuleFile(Utf8JsonWriter writer, IModuleData mod) + { + //Set object name + writer.WriteString("module_name", mod.ModuleName); + //Modified date + writer.WriteString("modifed_date", DateTime.UtcNow); + } + + private static void WriteCommitHistory(Utf8JsonWriter writer, Repository repo) + { + writer.WriteStartArray("commits"); + + //Write commit history for current repo + foreach (Commit commit in repo.Head.Commits) + { + writer.WriteStartObject(); + + WriteSingleCommit(writer, commit); + + writer.WriteEndObject(); + } + + writer.WriteEndArray(); + + //Write tag history for current repo + writer.WriteStartArray("tags"); + + foreach (Tag tag in repo.Tags) + { + //clamp message length and ellipsis if too long + string? message = tag.Annotation?.Message; + if (message != null && message.Length > 120) + { + message = $"{message[..120]}..."; + } + + writer.WriteStartObject(); + writer.WriteString("name", tag.FriendlyName); + writer.WriteString("sha", tag.Target.Sha); + writer.WriteString("message", message); + writer.WriteString("author", tag.Annotation?.Tagger.Name); + writer.WriteString("date", (tag.Annotation?.Tagger.When ?? default)); + writer.WriteEndObject(); + } + + writer.WriteEndArray(); + } + + private static void WriteSingleCommit(Utf8JsonWriter writer, Commit commit) + { + writer.WriteString("sha", commit.Sha); + writer.WriteString("message", commit.Message); + writer.WriteString("author", commit.Author.Name); + writer.WriteString("commiter", commit.Committer.Name); + writer.WriteString("date", commit.Committer.When); + writer.WriteString("message_short", commit.MessageShort); + } + + /// <summary> + /// Builds and writes the projects information to the <see cref="Utf8JsonWriter"/> + /// </summary> + /// <param name="project"></param> + /// <param name="writer"> + /// The <see cref="Utf8JsonWriter"/> to write the project + /// information to + /// </param> + /// <returns>A task that completes when the write operation has completed</returns> + private async Task WriteProjectInfoAsync(IModuleFileManager man, IProject project, string latestSha, string version, Utf8JsonWriter writer) + { + //Reload the project dom after execute because semversion may be updated after build step + if (project is ModuleProject mp) + { + await mp.LoadProjectDom(); + } + + writer.WriteString("name", project.ProjectName); + writer.WriteString("repo_url", project.ProjectData.RepoUrl); + writer.WriteString("description", project.ProjectData.Description); + writer.WriteString("version", version); + writer.WriteString("copyright", project.ProjectData.Copyright); + writer.WriteString("author", project.ProjectData.Authors); + writer.WriteString("product", project.ProjectData.Product); + writer.WriteString("company", project.ProjectData.CompanyName); + writer.WriteString("commit", latestSha); + //Write target framework if it exsits + writer.WriteString("target_framework", project.ProjectData["TargetFramework"]); + + //Start file array + writer.WriteStartArray("files"); + + //Get only tar files, do not include the sha files + foreach (FileInfo output in GetProjOutputFiles(man, project)) + { + //beging file object + writer.WriteStartObject(); + + writer.WriteString("name", output.Name); + writer.WriteString("path", $"{project.GetSafeProjectName()}/{output.Name}"); + writer.WriteString("date", output.LastWriteTimeUtc); + writer.WriteNumber("size", output.Length); + + //Compute the file hash + string hashHex = await output.ComputeFileHashStringAsync(); + writer.WriteString(config.HashFuncName, hashHex); + + //Path to sha-file + writer.WriteString("sha_file", $"{project.GetSafeProjectName()}/{output.Name}.{config.HashFuncName}"); + + writer.WriteEndObject(); + } + + writer.WriteEndArray(); + } + + private IEnumerable<FileInfo> GetProjOutputFiles(IModuleFileManager man, IProject project) + { + return man.GetArtifactOutputDir(project).EnumerateFiles("*.*", SearchOption.TopDirectoryOnly) + .Where(p => p.Extension != $".{config.HashFuncName}"); + } + + /// <summary> + /// Copies the source git archive tgz file to the output directory + /// </summary> + /// <param name="mod"></param> + /// <param name="man"></param> + /// <returns></returns> + private async Task<string?> CopySourceArchiveToOutput(IModuleData mod, IModuleFileManager man) + { + //Try to get a source archive in the module directory + string? archiveFile = Directory.EnumerateFiles(mod.Repository.Info.WorkingDirectory, config.SourceArchiveName, SearchOption.TopDirectoryOnly).FirstOrDefault(); + + //If archive is null ignore and continue + if(string.IsNullOrWhiteSpace(archiveFile)) + { + Log.Information("No archive file found for module {mod}", mod.ModuleName); + return null; + } + + Log.Information("Found source archive for module {mod}, copying to output...", mod.ModuleName); + + //Otherwise copy to output + byte[] archive = await File.ReadAllBytesAsync(archiveFile); + FileInfo output = await man.WriteFileAsync(ModuleFileType.Archive, archive); + + //Compute the hash of the file + await output.ComputeFileHashAsync(config.HashFuncName); + + if (SignEnabled) + { + //Sign the file if signing is enabled + await signer.SignFileAsync(output); + } + + return archiveFile; + } + } +}
\ No newline at end of file diff --git a/src/BuildStepFailedException.cs b/src/BuildStepFailedException.cs new file mode 100644 index 0000000..0b78bf3 --- /dev/null +++ b/src/BuildStepFailedException.cs @@ -0,0 +1,29 @@ +using System; + +namespace VNLib.Tools.Build.Executor +{ + sealed class BuildStepFailedException : Exception + { + public string? ArtifactName { get; set; } + + public BuildStepFailedException() + { } + + public BuildStepFailedException(string? message) : base(message) + { } + + + public BuildStepFailedException(string? message, Exception? innerException) : base(message, innerException) + { } + + public BuildStepFailedException(string? message, Exception? innerException, string name) : base(message, innerException) + { + ArtifactName = name; + } + + public BuildStepFailedException(string message, string artifactName):base(message) + { + this.ArtifactName = artifactName; + } + } +}
\ No newline at end of file diff --git a/src/Commands/BaseCommand.cs b/src/Commands/BaseCommand.cs new file mode 100644 index 0000000..6362c22 --- /dev/null +++ b/src/Commands/BaseCommand.cs @@ -0,0 +1,116 @@ +using System; +using System.IO; +using System.Threading; +using System.Threading.Tasks; + +using Typin; +using Typin.Console; +using Typin.Attributes; + +using VNLib.Tools.Build.Executor.Model; +using VNLib.Tools.Build.Executor.Constants; +using static VNLib.Tools.Build.Executor.Constants.Config; + +namespace VNLib.Tools.Build.Executor.Commands +{ + public abstract class BaseCommand(BuildPipeline pipeline, ConfigManager bm) : ICommand + { + [CommandOption("verbose", 'v', Description = "Prints verbose output")] + public bool Verbose { get; set; } + + [CommandOption("force", 'f', Description = "Forces the operation even if steps are required")] + public bool Force { get; set; } + + [CommandOption("modules", 'm', Description = "Only use the specified modules, comma separated list")] + public string? Modules { get; set; } + + [CommandOption("exclude", 'x', Description = "Ignores the specified modules, comma separated list")] + public string? Exclude { get; set; } + + [CommandOption("confirm", 'c', Description = "Wait for user input before continuing")] + public bool Confirm { get; set; } + + //Allow users to specify build directory + [CommandOption("build-dir", 'B', Description = "Sets the base build directory to execute operations in")] + public string BuildDir { get; set; } = Directory.GetCurrentDirectory(); + + [CommandOption("max-logs", 'L', Description = "Sets the maximum number of logs to keep")] + public int MaxLogs { get; set; } = 50; + + public IDirectoryIndex BuildIndex { get; private set; } = default!; + + public BuildConfig Config { get; private set; } = default!; + + + /* + * Base exec command does basic init and cleanup on startup + */ + + public virtual async ValueTask ExecuteAsync(IConsole console) + { + try + { + CancellationToken ct = console.GetCancellationToken(); + + //Init build index + BuildIndex = GetIndex(); + Config = await bm.GetOrCreateConfig(BuildIndex, false); + + //Cleanup log files on init + TrimLogs(BuildIndex, MaxLogs); + + string[] modules = Modules?.Split(',') ?? []; + string[] exclude = Exclude?.Split(',') ?? []; + + //Always load the pipeline + await pipeline.LoadAsync(Config, modules, exclude, Feeds); + + if (Confirm) + { + console.Output.WriteLine("---- Pipeline loaded. Press any key to continue ----"); + await console.Input.ReadLineAsync(ct); + + await Task.Delay(100, ct); + ct.ThrowIfCancellationRequested(); + } + + //Exec steps then exit + await ExecStepsAsync(console); + } + catch (OperationCanceledException) + { + console.WithForegroundColor(ConsoleColor.Red, static o => o.Output.WriteLine("Operation cancelled")); + } + catch(BuildStepFailedException be) + { + console.WithForegroundColor(ConsoleColor.Red, o => o.Output.WriteLine("FATAL: Build step failed on module {0}, msg -> {1}", be.ArtifactName, be.Message)); + } + } + + public abstract ValueTask ExecStepsAsync(IConsole console); + + public abstract IFeedManager[] Feeds { get; } + + private Dirs GetIndex() => new() + { + BaseDir = new(BuildDir), + BuildDir = BuildDirs.GetOrCreateDir(BUILD_DIR_NAME), + LogDir = BuildDirs.GetOrCreateDir(LOG_DIR_NAME), + ScratchDir = BuildDirs.GetOrCreateDir(SCRATCH_DIR), + SumDir = BuildDirs.GetOrCreateDir(SUM_DIR), + OutputDir = BuildDirs.GetOrCreateDir(OUTPUT_DIR) + }; + +#nullable disable + protected sealed class Dirs : IDirectoryIndex + { + public DirectoryInfo BaseDir { get; set; } + public DirectoryInfo BuildDir { get; set; } + public DirectoryInfo LogDir { get; set; } + public DirectoryInfo ScratchDir { get; set; } + public DirectoryInfo SumDir { get; set; } + public DirectoryInfo OutputDir { get; set; } + } +#nullable enable + } +}
\ No newline at end of file diff --git a/src/Commands/BuildCommand.cs b/src/Commands/BuildCommand.cs new file mode 100644 index 0000000..c1d9828 --- /dev/null +++ b/src/Commands/BuildCommand.cs @@ -0,0 +1,66 @@ +using System; +using System.Threading; +using System.Threading.Tasks; + +using Typin.Console; +using Typin.Attributes; + +using VNLib.Tools.Build.Executor.Model; +using VNLib.Tools.Build.Executor.Constants; + +namespace VNLib.Tools.Build.Executor.Commands +{ + + [Command("build", Description = "Executes a build operation in pipeline")] + public class BuildCommand(BuildPipeline pipeline, ConfigManager bm) : BaseCommand(pipeline, bm) + { + + [CommandOption("no-delay", 'S', Description = "Skips any built-in delay/wait")] + public bool SkipDelay { get; set; } = false; + + public override async ValueTask ExecStepsAsync(IConsole console) + { + CancellationToken cancellation = console.GetCancellationToken(); + + console.Output.WriteLine("Starting build pipeline. Checking for source code changes"); + + if (Force) + { + console.WithForegroundColor(ConsoleColor.Yellow, static o => o.Output.WriteLine("Forcing build step")); + } + + //Check for source code changes + bool changed = await pipeline.CheckForChangesAsync(); + + //continue build + if (!Force && !changed) + { + console.WithForegroundColor(ConsoleColor.Green, static o => o.Output.WriteLine("No source code changes detected. Skipping build step")); + return; + } + + if (Confirm) + { + console.Output.WriteLine("Press any key to continue..."); + await console.Input.ReadLineAsync(cancellation); + cancellation.ThrowIfCancellationRequested(); + } + else if(!SkipDelay) + { + //wait for 10 seconds + for (int i = 10; i > 0; i--) + { + string seconds = i > 1 ? "seconds" : "second"; + console.Output.WriteLine($"Starting build step in {i} {seconds}"); + await Task.Delay(1000, cancellation); + } + } + + await pipeline.DoStepBuild(Force); + + console.WithForegroundColor(ConsoleColor.Green, static o => o.Output.WriteLine("Build completed successfully")); + } + + public override IFeedManager[] Feeds => []; + } +}
\ No newline at end of file diff --git a/src/Commands/CleanCommand.cs b/src/Commands/CleanCommand.cs new file mode 100644 index 0000000..6570b31 --- /dev/null +++ b/src/Commands/CleanCommand.cs @@ -0,0 +1,31 @@ +using System; +using System.Threading.Tasks; + +using Typin.Console; +using Typin.Attributes; + +using VNLib.Tools.Build.Executor.Model; +using VNLib.Tools.Build.Executor.Constants; + +namespace VNLib.Tools.Build.Executor.Commands +{ + + [Command("clean", Description = "Cleans the build pipeline")] + public sealed class CleanCommand(BuildPipeline pipeline, ConfigManager cfg) : BaseCommand(pipeline, cfg) + { + public override async ValueTask ExecStepsAsync(IConsole console) + { + console.Output.WriteLine("Begining clean step"); + + await pipeline.DoStepCleanAsync(); + + //Cleanup the sum dir and scratch dir + Config.Index.SumDir.Delete(true); + Config.Index.ScratchDir.Delete(true); + + console.WithForegroundColor(ConsoleColor.Green, o => o.Output.WriteLine("Pipeline cleaned")); + } + + public override IFeedManager[] Feeds => []; + } +}
\ No newline at end of file diff --git a/src/Commands/PublishCommand.cs b/src/Commands/PublishCommand.cs new file mode 100644 index 0000000..4179f86 --- /dev/null +++ b/src/Commands/PublishCommand.cs @@ -0,0 +1,80 @@ + +using System; +using System.Linq; +using System.Threading.Tasks; + +using Typin.Console; +using Typin.Attributes; + +using VNLib.Tools.Build.Executor.Model; +using VNLib.Tools.Build.Executor.Constants; + +namespace VNLib.Tools.Build.Executor.Commands +{ + [Command("publish", Description = "Runs publishig build steps on a completed build")] + public sealed class PublishCommand(BuildPipeline pipeline, ConfigManager bm) : BaseCommand(pipeline, bm) + { + [CommandOption("upload-path", 'p', Description = "The path to upload the build artifacts")] + public string? UploadPath { get; set; } + + [CommandOption("sign", 's', Description = "Enables gpg signing of build artifacts")] + public bool Sign { get; set; } = false; + + [CommandOption("gpg-key", 'k', Description = "Optional key to use when signing, otherwise uses the GPG default signing key")] + public string? GpgKey { get; set; } + + [CommandOption("sleet-path", 'F', Description = "Specifies the Sleet feed index path")] + public string? SleetPath { get; set; } + + [CommandOption("dry-run", 'd', Description = "Executes all publish steps without pushing the changes to the remote server")] + public bool DryRun { get; set; } + + [CommandOption("output", 'o', Description = "Specifies the output directory for the published modules")] + public string? CustomOutDir { get; set; } + + public override async ValueTask ExecStepsAsync(IConsole console) + { + //Specify custom output dir + (base.Config.Index as Dirs)!.OutputDir = BuildDirs.GetOrCreateDir(Constants.Config.OUTPUT_DIR, CustomOutDir); + + IUploadManager? uploads = MinioUploadManager.Create(UploadPath); + IFeedManager? feed = Feeds.FirstOrDefault(); + + //Optional gpg signer for signing published artifacts + BuildPublisher pub = new(Config, new GpgSigner(Sign, GpgKey)); + + console.WithForegroundColor(ConsoleColor.DarkGreen, static o => o.Output.WriteLine("Publishing modules")); + + //Run publish steps + await pipeline.OnPublishingAsync().ConfigureAwait(false); + + console.WithForegroundColor(ConsoleColor.DarkGreen, static o => o.Output.WriteLine("Preparing module output for upload")); + + //Prepare the output + await pipeline.PrepareOutputAsync(pub).ConfigureAwait(false); + + if(uploads is null) + { + console.WithForegroundColor(ConsoleColor.DarkYellow, static o => o.Output.WriteLine("No upload path specified. Skipping upload")); + console.WithForegroundColor(ConsoleColor.Green, static o => o.Output.WriteLine("Upload build complete")); + return; + } + + //Run upload + await pipeline.ManualUpload(pub, uploads).ConfigureAwait(false); + + //Publish feeds + if (feed is not null) + { + console.WithForegroundColor(ConsoleColor.DarkGreen, static o => o.Output.WriteLine("Uploading feeds...")); + + //Exec feed upload + await uploads.UploadDirectoryAsync(feed.FeedOutputDir); + } + + console.WithForegroundColor(ConsoleColor.Green, static o => o.Output.WriteLine("Upload build complete")); + } + + public override IFeedManager[] Feeds => SleetPath is null ? [] : [SleetFeedManager.GetSleetFeed(SleetPath)]; + } +}
\ No newline at end of file diff --git a/src/Commands/TestCommand.cs b/src/Commands/TestCommand.cs new file mode 100644 index 0000000..d64fe64 --- /dev/null +++ b/src/Commands/TestCommand.cs @@ -0,0 +1,31 @@ +using System; +using System.Threading.Tasks; + +using Typin.Console; +using Typin.Attributes; + +using VNLib.Tools.Build.Executor.Model; +using VNLib.Tools.Build.Executor.Constants; + +namespace VNLib.Tools.Build.Executor.Commands +{ + [Command("test", Description = "Executes tests steps within the pipline for all loaded modules")] + public sealed class TestCommand(BuildPipeline pipeline, ConfigManager cfg) : BaseCommand(pipeline, cfg) + { + private readonly BuildPipeline _pipeline = pipeline; + + [CommandOption("--no-fail", Description = "Exit testing on the first test failure")] + public bool FailOnTestFail { get; set; } = true; + + public override async ValueTask ExecStepsAsync(IConsole console) + { + console.Output.WriteLine("Begining test step"); + + await _pipeline.ExecuteTestsAsync(FailOnTestFail); + + console.WithForegroundColor(ConsoleColor.Green, o => o.Output.WriteLine("Pipeline tests compled")); + } + + public override IFeedManager[] Feeds => []; + } +}
\ No newline at end of file diff --git a/src/Commands/TestDisplayCommand.cs b/src/Commands/TestDisplayCommand.cs new file mode 100644 index 0000000..c716ada --- /dev/null +++ b/src/Commands/TestDisplayCommand.cs @@ -0,0 +1,22 @@ +using System.Threading.Tasks; + +using Typin.Console; +using Typin.Attributes; + +using VNLib.Tools.Build.Executor.Model; +using VNLib.Tools.Build.Executor.Constants; + +namespace VNLib.Tools.Build.Executor.Commands +{ + [Command("display", Description = "Test command for debugging")] + public sealed class TestDisplayCommand(BuildPipeline pipeline, ConfigManager bm) : BaseCommand(pipeline, bm) + { + public override IFeedManager[] Feeds => []; + + public override async ValueTask ExecStepsAsync(IConsole console) + { + console.Output.WriteLine("Press any key to exit..."); + await console.Input.ReadLineAsync(console.GetCancellationToken()); + } + } +}
\ No newline at end of file diff --git a/src/Commands/UpdateCommand.cs b/src/Commands/UpdateCommand.cs new file mode 100644 index 0000000..e3bdf9c --- /dev/null +++ b/src/Commands/UpdateCommand.cs @@ -0,0 +1,25 @@ +using System; +using System.Threading.Tasks; + +using Typin.Console; +using Typin.Attributes; + +using VNLib.Tools.Build.Executor.Model; +using VNLib.Tools.Build.Executor.Constants; + +namespace VNLib.Tools.Build.Executor.Commands +{ + [Command("update", Description = "Runs the build steps for updating application soure code")] + public sealed class UpdateCommand(BuildPipeline pipeline, ConfigManager cfg) : BaseCommand(pipeline, cfg) + { + public override async ValueTask ExecStepsAsync(IConsole console) + { + //Run the update step + await pipeline.DoStepUpdateSource(); + + console.WithForegroundColor(ConsoleColor.Green, o => o.Output.WriteLine("Source update complete")); + } + + public override IFeedManager[] Feeds => []; + } +}
\ No newline at end of file diff --git a/src/Constants/BuildConfig.cs b/src/Constants/BuildConfig.cs new file mode 100644 index 0000000..39758ea --- /dev/null +++ b/src/Constants/BuildConfig.cs @@ -0,0 +1,77 @@ +using System.Text.Json.Serialization; + +using Semver; + +using VNLib.Tools.Build.Executor.Model; + +namespace VNLib.Tools.Build.Executor.Constants +{ + public sealed class BuildConfig + { + [JsonPropertyName("soure_file_extensions")] + public string[] SourceFileEx { get; set; } = [ + "c", + "cpp", + "cxx", + "h", + "hpp", + "cs", + "proj", + "sln", + "ts", + "js", + "java", + "json", + "yaml", + "yml", + ]; + + [JsonPropertyName("excluded_dirs")] + public string[] ExcludedSourceDirs { get; set; } = [ + "bin", + "obj", + "packages", + "node_modules", + "dist", + "build", + "out", + "target", + ]; + + [JsonPropertyName("default_sha_method")] + public string HashFuncName { get; set; } = "sha256"; + + [JsonPropertyName("head_file_name")] + public string HeadFileName { get; set; } = "@latest"; + + [JsonPropertyName("module_task_file_name")] + public string ModuleTaskFileName { get; set; } = "Module.Taskfile.yaml"; + + [JsonPropertyName("main_taskfile_name")] + public string MainTaskFileName { get; set; } = "build.taskfile.yaml"; + + [JsonPropertyName("output_file_type")] + public string OutputFileType { get; set; } = "*.tgz"; + + [JsonPropertyName("task_exe_name")] + public string TaskExeName { get; set; } = "task"; + + [JsonPropertyName("source_archive_name")] + public string SourceArchiveName { get; set; } = "archive.tgz"; + + [JsonPropertyName("source_archive_format")] + public string SourceArchiveFormat { get; set; } = "tgz"; + + [JsonPropertyName("project_bin_dir")] + public string ProjectBinDir { get; set; } = "bin"; + + [JsonPropertyName("default_ci_version")] + public string DefaultCiVersion { get; set; } = "0.1.0"; + + [JsonPropertyName("semver_style")] + public SemVersionStyles SemverStyle { get; set; } + + [JsonIgnore] + public IDirectoryIndex Index { get; set; } = default!; + } +}
\ No newline at end of file diff --git a/src/Constants/BuildDirs.cs b/src/Constants/BuildDirs.cs new file mode 100644 index 0000000..78609f5 --- /dev/null +++ b/src/Constants/BuildDirs.cs @@ -0,0 +1,35 @@ +using System; +using System.IO; + +namespace VNLib.Tools.Build.Executor.Constants +{ + internal static class BuildDirs + { + + private static DirectoryInfo GetProjectDir() + { + //See if dir was specified on command line + string[] args = Environment.GetCommandLineArgs(); + + //Get the build dir + DirectoryInfo dir = new(args.Length > 1 && Directory.Exists(args[1]) ? args[1] : Directory.GetCurrentDirectory()); + + if (!dir.Exists) + { + dir.Create(); + } + return dir; + } + + public static DirectoryInfo GetOrCreateDir(string @default, string? other = null) + { + //Get the scratch dir + DirectoryInfo logDir = new(Path.Combine(GetProjectDir().FullName, other?? @default)); + if (!logDir.Exists) + { + logDir.Create(); + } + return logDir; + } + } +}
\ No newline at end of file diff --git a/src/Constants/Config.cs b/src/Constants/Config.cs new file mode 100644 index 0000000..fc35928 --- /dev/null +++ b/src/Constants/Config.cs @@ -0,0 +1,95 @@ +using System; +using System.IO; +using System.Linq; + +using Serilog; +using Serilog.Core; + +using VNLib.Tools.Build.Executor.Model; + +namespace VNLib.Tools.Build.Executor.Constants +{ + + internal static class Config + { + + //relative local directores to the project root + public const string BUILD_DIR_NAME = ".build"; + public const string LOG_DIR_NAME = ".build/log"; + public const string BUILD_CONFIG = "build.conf.json"; + public const string SCRATCH_DIR = ".build/scratch"; + public const string SUM_DIR = ".build/sums"; + public const string OUTPUT_DIR = ".build/output"; + public const string SLEET_DIR = ".build/feed"; + + /// <summary> + /// Gets the system wide <see cref="Logger"/> log writer instance + /// </summary> + public static Logger Log { get; } = GetLog(); + + const string Template = "{Message:lj}{NewLine}{Exception}"; + + private static Logger GetLog() + { + string[] args = Environment.GetCommandLineArgs(); + + LoggerConfiguration conf = new(); + + if (args.Contains("-v") || args.Contains("--verbose")) + { + //Check for verbose logging level + conf.MinimumLevel.Verbose(); + } + else if (args.Contains("-d") || args.Contains("--debug")) + { + //Check for debug + conf.MinimumLevel.Debug(); + } + else + { + //Default information level + conf.MinimumLevel.Information(); + } + + //Create a console logger unless the silent flag is set + if (!args.Contains("-s")) + { + conf.WriteTo.Console(outputTemplate: Template); + } + + //Creat the new log file + string logFilePath = Path.Combine(LOG_DIR_NAME, $"{DateTimeOffset.Now.ToUnixTimeSeconds()}-log.txt"); + + //Setup the log file output + conf.WriteTo.File(logFilePath, outputTemplate: Template); + + return conf.CreateLogger(); + } + + /// <summary> + /// Cleans up old build log files, so that only 100 log files remain in the log directory + /// </summary> + public static void TrimLogs(IDirectoryIndex dirIndex, int maxLogs) + { + try + { + //Get all log files in the log directory and cleanup any files after the max log count + FileInfo[] toDelete = dirIndex.LogDir.EnumerateFiles("*.txt", SearchOption.TopDirectoryOnly) + .OrderByDescending(static f => f.LastWriteTimeUtc) + .Skip(maxLogs) + .ToArray(); + + foreach (FileInfo file in toDelete) + { + file.Delete(); + } + + Log.Debug("Cleaned {file} log files", toDelete.Length); + } + catch (Exception ex) + { + Log.Warning(ex, "Failed to cleanup log files"); + } + } + } +}
\ No newline at end of file diff --git a/src/Constants/ConfigManager.cs b/src/Constants/ConfigManager.cs new file mode 100644 index 0000000..3a0b295 --- /dev/null +++ b/src/Constants/ConfigManager.cs @@ -0,0 +1,41 @@ +using System.IO; +using System.Text.Json; +using System.Threading.Tasks; + +using Semver; + +using VNLib.Tools.Build.Executor.Model; + +namespace VNLib.Tools.Build.Executor.Constants +{ + public class ConfigManager(SemVersionStyles semver) + { + public async Task<BuildConfig> GetOrCreateConfig(IDirectoryIndex index, bool overwrite) + { + //Get the config file path + string configFilePath = Path.Combine(index.BuildDir.FullName, Config.BUILD_CONFIG); + + BuildConfig? data = new() + { + SemverStyle = semver + }; + + //If the file doesnt exist or we want to overwrite it + if (!File.Exists(configFilePath) || overwrite) + { + //Create a new config file + await using FileStream fs = File.Create(configFilePath); + await JsonSerializer.SerializeAsync(fs, data); + } + else + { + await using FileStream fs = File.OpenRead(configFilePath); + data = await JsonSerializer.DeserializeAsync<BuildConfig>(fs); + } + + data!.Index = index; + + return data; + } + } +}
\ No newline at end of file diff --git a/src/Constants/ConsoleCancelToken.cs b/src/Constants/ConsoleCancelToken.cs new file mode 100644 index 0000000..cc25feb --- /dev/null +++ b/src/Constants/ConsoleCancelToken.cs @@ -0,0 +1,29 @@ +using System; +using System.Threading; + +namespace VNLib.Tools.Build.Executor.Constants +{ + internal sealed class ConsoleCancelToken : IDisposable + { + private readonly CancellationTokenSource _cts = new(); + + public CancellationToken Token => _cts.Token; + + public ConsoleCancelToken() => Console.CancelKeyPress += OnCancel; + + private void OnCancel(object? sender, ConsoleCancelEventArgs e) + { + _cts.Cancel(); + e.Cancel = true; + } + + public void Dispose() + { + //Unsubscribe from event + Console.CancelKeyPress -= OnCancel; + _cts.Dispose(); + + GC.SuppressFinalize(this); + } + } +}
\ No newline at end of file diff --git a/src/Constants/Utils.cs b/src/Constants/Utils.cs new file mode 100644 index 0000000..6047c35 --- /dev/null +++ b/src/Constants/Utils.cs @@ -0,0 +1,151 @@ +using System; +using System.Threading; +using System.Diagnostics; +using System.Threading.Tasks; +using System.Collections.Generic; +using System.Runtime.CompilerServices; + +using static VNLib.Tools.Build.Executor.Constants.Config; + +namespace VNLib.Tools.Build.Executor.Constants +{ + + internal static class Utils + { + + /// <summary> + /// Runs a process by its name/exe file path, and writes its stdout/stderr to + /// the default build log + /// </summary> + /// <param name="process">The name of the process to run</param> + /// <param name="args">CLI arguments to pass to the process</param> + /// <returns>The process exit code</returns> + public static async Task<int> RunProcessAsync(string process, string? workingDir, string[] args, IReadOnlyDictionary<string, string>? env = null) + { + //Init new console cancellation token + using ConsoleCancelToken ctToken = new(); + + ProcessStartInfo psi = new(process) + { + //Redirect streams + RedirectStandardError = true, + RedirectStandardOutput = true, + CreateNoWindow = true, + //Create a child process, not shell + UseShellExecute = false, + WorkingDirectory = workingDir ?? string.Empty, + }; + + if (env != null) + { + //Add all env variables to process + foreach (KeyValuePair<string, string> kv in env) + { + psi.Environment.Add(kv.Key, kv.Value); + } + } + + //Add arguments + foreach (string arg in args) + { + psi.ArgumentList.Add(arg); + } + + using Process proc = new(); + proc.StartInfo = psi; + + //Start the process + proc.Start(); + + Log.Debug("Starting process {proc}, with args {args}", proc.ProcessName, args); + Console.WriteLine(); + + //Log std out + Task stdout = LogStdOutAsync(proc, ctToken.Token); + Task stdErr = LogStdErrAsync(proc, ctToken.Token); + + //Wait for the process to exit + Task wfe = proc.WaitForExitAsync(ctToken.Token); + + //Wait for stderr/out/proc to exit + await Task.WhenAll(stdout, stdErr, wfe); + + Console.WriteLine(); + Log.Debug("[CHILD]:{id}:{p} exited w/ code {code}", proc.ProcessName, proc.Id, proc.ExitCode); + + //Return status code + return proc.ExitCode; + } + + private static async Task LogStdOutAsync(Process psi, CancellationToken cancellation) + { + try + { + string procName = psi.ProcessName; + int id = psi.Id; + + do + { + //Read lines from the process + string? line = await psi.StandardOutput.ReadLineAsync(cancellation); + + if (line == null) + { + break; + } + + //Print to log file + Console.WriteLine(line); + } while (!psi.HasExited); + } + catch (Exception ex) + { + Log.Error(ex, "An exception was raised while reading the process standard output"); + } + } + + private static async Task LogStdErrAsync(Process psi, CancellationToken cancellation) + { + try + { + string procName = psi.ProcessName; + int id = psi.Id; + + do + { + //Read lines from the process + string? line = await psi.StandardError.ReadLineAsync(cancellation); + + if (line == null) + { + break; + } + + //Print to log file + Console.WriteLine(line); + } while (!psi.HasExited); + } + catch (Exception ex) + { + Log.Error(ex, "An exception was raised while reading the process standard output"); + } + } + + + /// <summary> + /// Throws a <see cref="BuildStepFailedException"/> if the value + /// of <paramref name="status"/> is false + /// </summary> + /// <param name="status">If false throws exception</param> + /// <param name="message">The message to display</param> + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static void ThrowIfStepFailed(bool status, string message, string artifactName) + { + if (!status) + { + throw new BuildStepFailedException(message, artifactName); + } + } + + } +}
\ No newline at end of file diff --git a/src/Extensions/BuildExtensions.cs b/src/Extensions/BuildExtensions.cs new file mode 100644 index 0000000..24af05e --- /dev/null +++ b/src/Extensions/BuildExtensions.cs @@ -0,0 +1,150 @@ +using System; +using System.IO; +using System.Linq; +using System.Threading.Tasks; +using System.Collections.Generic; +using System.Security.Cryptography; + +using LibGit2Sharp; + +using Semver; + +using VNLib.Tools.Build.Executor.Model; + +namespace VNLib.Tools.Build.Executor.Extensions +{ + internal static class BuildExtensions + { + + /// <summary> + /// Determines if the file exists within the current directory + /// </summary> + /// <param name="dir"></param> + /// <param name="fileName">The name of the file to search for</param> + /// <returns>True if the file exists, false otherwise</returns> + public static bool FileExists(this DirectoryInfo dir, string fileName) => File.Exists(Path.Combine(dir.FullName, fileName)); + + /// <summary> + /// Determines if a child directory exists + /// </summary> + /// <param name="dir"></param> + /// <param name="dirName">The name of the directory to check for</param> + /// <returns>True if the directory exists</returns> + public static bool ChildExists(this DirectoryInfo dir, string dirName) => Directory.Exists(Path.Combine(dir.FullName, dirName)); + + /// <summary> + /// Deletes a child directory + /// </summary> + /// <param name="dir"></param> + /// <param name="dirName">The name of the directory to delete</param> + /// <param name="recurse">Recursive delete, delete all child items</param> + public static void DeleteChild(this DirectoryInfo dir, string dirName, bool recurse = true) => Directory.Delete(Path.Combine(dir.FullName, dirName), recurse); + + /// <summary> + /// Creates a child directory within the current directory + /// </summary> + /// <param name="dir"></param> + /// <param name="name">The name of the child directory</param> + public static DirectoryInfo CreateChild(this DirectoryInfo dir, string name) => Directory.CreateDirectory(Path.Combine(dir.FullName, name)); + + /// <summary> + /// Computes the SHA256 hash of the current file and writes the hash to + /// a filename.sha256 hexadecimal text file + /// </summary> + /// <param name="file"></param> + /// <returns>A task the completes when the file hash has been produced in the output directory</returns> + public static async Task ComputeFileHashAsync(this FileInfo file, string hashName) + { + string outputName = $"{file.FullName}.{hashName}"; + + //convert the hash to hexadecimal + string hex = await ComputeFileHashStringAsync(file); + + //Write the hex hash to the output file + await File.WriteAllTextAsync(outputName, hex); + } + + /// <summary> + /// Computes the SHA256 hash of the current file and returns the file hash as a hexadecimal string + /// </summary> + /// <param name="file"></param> + /// <returns>A task the completes when the file hash has been produced in the output directory</returns> + public static async Task<string> ComputeFileHashStringAsync(this FileInfo file) + { + using SHA256 alg = SHA256.Create(); + + //Open the output file to read the file data to compute hash + await using FileStream input = file.OpenRead(); + + //Compute hash + byte[] hash = await alg.ComputeHashAsync(input); + + //convert the hash to hexadecimal + return Convert.ToHexString(hash); + } + + + public static Task RunAllAsync<T>(this IEnumerable<T> workCol, Func<T, Task> cb) + { + Task[] tasks = workCol.Select(cb).ToArray(); + return Task.WhenAll(tasks); + } + + /// <summary> + /// Gets the module's version based on the latest tag and the number of commits since the last tag + /// that supports pre-release/semver + /// </summary> + /// <param name="mod"></param> + /// <param name="style">The version style</param> + /// <returns>The ci build/version number</returns> + public static string GetModuleCiVersion(this IModuleData mod, string defaultCiVersion, SemVersionStyles style) + { + int ciNumber = 0; + SemVersion baseVersion; + + //Get latest version tag from git + Tag? vTag = mod.Repository.Tags.OrderByDescending(p => SemVersion.Parse(p.FriendlyName, style)).FirstOrDefault(); + + //Find the number of commits since the last tag + if (vTag != null) + { + //Get the number of commits since the last tag + baseVersion = SemVersion.Parse(vTag.FriendlyName, style); + + //Search through commits till we can find the commit that matches the tag + Commit[] commits = mod.Repository.Commits.ToArray(); + + for (; ciNumber < commits.Length; ciNumber++) + { + if (commits[ciNumber].Sha == vTag.Target.Sha) + { + break; + } + } + } + else + { + //No tags, so just use the number of commits + ciNumber = mod.Repository.Commits.Count() - 1; + baseVersion = SemVersion.Parse(defaultCiVersion, style); + } + + //If there are commits, then increment the version prerelease + if (ciNumber > 0) + { + //Increment the version + baseVersion = baseVersion.WithPrerelease($"ci{ciNumber:0000}"); + } + + return baseVersion.ToString(); + } + + public static bool IsFileIgnored(this Repository repo, string file) + { + FileStatus status = repo.RetrieveStatus(file); + + //If the leaf project is ignored, skip it + return status.HasFlag(FileStatus.Ignored) || status.HasFlag(FileStatus.Nonexistent); + } + } +}
\ No newline at end of file diff --git a/src/Extensions/FileManagerExtensions.cs b/src/Extensions/FileManagerExtensions.cs new file mode 100644 index 0000000..6f497e7 --- /dev/null +++ b/src/Extensions/FileManagerExtensions.cs @@ -0,0 +1,135 @@ +using System; +using System.IO; +using System.Linq; +using System.Text.Json; +using System.Threading.Tasks; +using System.Collections.Generic; +using System.Runtime.CompilerServices; + +using VNLib.Tools.Build.Executor.Model; +using VNLib.Tools.Build.Executor.Constants; +using static VNLib.Tools.Build.Executor.Constants.Config; + +namespace VNLib.Tools.Build.Executor.Extensions +{ + + internal static class FileManagerExtensions + { + + private static readonly ConditionalWeakTable<IProject, object> _projectHashes = new(); + + /// <summary> + /// Gets all external dependencies for the current module + /// </summary> + /// <param name="module"></param> + /// <returns>An array of project names of all the dependencies outside a given module</returns> + public static string[] GetExternalDependencies(this IModuleData module) + { + /* + * We need to get all child project dependencies that rely on projects + * outside of the current module. + * + * This assumes all projects within this model are properly linked + * and assumed to be build together, as is 99% of the case, otherwise + * custom build impl will happen at the Task level + */ + + //Get the project file names contained in the current module + string[] selfProjects = module.Projects.Select(static p => p.ProjectFile.Name).ToArray(); + + return module.Projects.SelectMany(static p => p.GetDependencies()).Where(dep => !selfProjects.Contains(dep)).ToArray(); + } + + /// <summary> + /// Determines if any source files have changed + /// </summary> + /// <param name="commit">The most recent commit hash</param> + public static async Task CheckSourceChangedAsync(this IModuleFileManager manager, IProject project, BuildConfig config, string commit) + { + //Compute current sum + string sum = await project.GetSourceFileHashAsync(config); + + //Old sum file exists + byte[]? sumData = await manager.ReadCheckSumAsync(project); + + //Try to read the old sum file + if (sumData != null) + { + //Parse sum file + using JsonDocument sumDoc = JsonDocument.Parse(sumData); + + //Get the sum + string? hexSum = sumDoc.RootElement.GetProperty("sum").GetString(); + + //Confirm the current sum and the found sum are equal + if (sum.Equals(hexSum, StringComparison.OrdinalIgnoreCase)) + { + Log.Verbose("Project {p} source is {up}", project.ProjectName, "up-to-date"); + + //Project source is up-to-date + project.UpToDate = true; + + //No changes made + return; + } + } + + Log.Verbose("Project {p} source is {up}", project.ProjectName, "changed"); + + //Store sum change + object sumChange = new Dictionary<string, string>() + { + { "sum", sum }, + { "commit", commit }, + { "modified", DateTimeOffset.UtcNow.ToString("s") } + }; + + //Store sum change for later + _projectHashes.Add(project, sumChange); + + project.UpToDate = false; + } + + /// <summary> + /// Creates the module's output directory + /// </summary> + /// <param name="manager"></param> + public static void CreateOutput(this IModuleFileManager manager) + { + //Create output directory for solution + _ = Directory.CreateDirectory(manager.OutputDir); + } + + /// <summary> + /// Deletes the output's module directory + /// </summary> + /// <param name="manager"></param> + public static void CleanOutput(this IModuleFileManager manager) + { + //Delete output directory for solution + if (Directory.Exists(manager.OutputDir)) + { + Directory.Delete(manager.OutputDir, true); + } + } + + + /// <summary> + /// Writes the source file checksum change to the project's sum file + /// </summary> + /// <param name="project"></param> + /// <param name="project">The project to write the checksum for</param> + /// <returns>A task that resolves when the sum file has been updated</returns> + public static Task CommitSumChangeAsync(this IModuleFileManager manager, IProject project) + { + if (!_projectHashes.TryGetValue(project, out object? sumChange)) + { + return Task.CompletedTask; + } + + byte[] sumData = JsonSerializer.SerializeToUtf8Bytes(sumChange); + + return manager.WriteChecksumAsync(project, sumData); + } + } +}
\ No newline at end of file diff --git a/src/Extensions/ProjectExtensions.cs b/src/Extensions/ProjectExtensions.cs new file mode 100644 index 0000000..a24e6c5 --- /dev/null +++ b/src/Extensions/ProjectExtensions.cs @@ -0,0 +1,128 @@ + +using System; +using System.IO; +using System.Linq; +using System.Threading.Tasks; +using System.Collections.Generic; +using System.Security.Cryptography; + +using VNLib.Tools.Build.Executor.Model; +using VNLib.Tools.Build.Executor.Constants; + +namespace VNLib.Tools.Build.Executor.Extensions +{ + + internal static class ProjectExtensions + { + + /// <summary> + /// Gets the project dependencies for the given project + /// </summary> + /// <param name="project"></param> + /// <returns>The list of project dependencies</returns> + public static string[] GetDependencies(this IProject project) + { + //Get the project file names (not paths) that are dependencies + return project.ProjectData.GetProjectRefs().Select(static r => Path.GetFileName(r)).ToArray(); + } + + private static bool IsSourceFile(BuildConfig conf, string fileName) + { + for (int i = 0; i < conf.SourceFileEx.Length; i++) + { + if (fileName.EndsWith(conf.SourceFileEx[i])) + { + return true; + } + } + return false; + } + + private static bool IsExcludedDir(BuildConfig conf, string path) + { + for (int i = 0; i < conf.ExcludedSourceDirs.Length; i++) + { + if (path.Contains(conf.ExcludedSourceDirs[i])) + { + return true; + } + } + return false; + } + + public static IEnumerable<FileInfo> GetProjectBuildFiles(this IProject project, BuildConfig config) + { + //See if an output dir is specified + string? outDir = project.ProjectData["output_dir"] ?? project.ProjectData["output"]; + + //If an output dir is specified, only get files from that dir + if(!string.IsNullOrWhiteSpace(outDir)) + { + //realtive file path + outDir = Path.Combine(project.WorkingDir.FullName, outDir); + + return new DirectoryInfo(outDir).EnumerateFiles(config.OutputFileType, SearchOption.TopDirectoryOnly); + } + else + { + return project.WorkingDir.EnumerateFiles(config.OutputFileType, SearchOption.AllDirectories); + } + } + + /// <summary> + /// Gets the sha256 hash of all the source files within the project + /// </summary> + /// <returns>A task that resolves the hexadecimal string of the sha256 hash of all the project source files</returns> + public static async Task<string> GetSourceFileHashAsync(this IProject project, BuildConfig config) + { + //Get all + FileInfo[] sourceFiles = project.WorkingDir!.EnumerateFiles("*.*", SearchOption.AllDirectories) + //Get all source files, c/c#/c++ source files, along with .xproj files (project files) + .Where(n => IsSourceFile(config, n.Name)) + //Exclude the obj intermediate output dir + .Where(f => !IsExcludedDir(config, f.DirectoryName ?? "")) + .ToArray(); + + //Get a scratch file to append file source code to + await using FileStream scratch = new( + $"{config.Index.ScratchDir.FullName}/{Path.GetRandomFileName()}", + FileMode.OpenOrCreate, + FileAccess.ReadWrite, + FileShare.None, + 8192, + FileOptions.DeleteOnClose + ); + + //Itterate over all source files + foreach (FileInfo sourceFile in sourceFiles) + { + //Open the source file stream + await using FileStream source = sourceFile.OpenRead(); + + //Append the data to the file stream + await source.CopyToAsync(scratch); + } + + //Flush the stream to disk and roll back to start + await scratch.FlushAsync(); + scratch.Seek(0, SeekOrigin.Begin); + + byte[] hash; + + //Create a sha 256 hash of the file + using (SHA256 alg = SHA256.Create()) + { + //Hash the file + hash = await alg.ComputeHashAsync(scratch); + } + + //Get hex of the hash + return Convert.ToHexString(hash); + } + + public static string GetSafeProjectName(this IProject project) + { + return project.ProjectName.Replace('/', '-').Replace('\\','-'); + } + } +}
\ No newline at end of file diff --git a/src/GpgSigner.cs b/src/GpgSigner.cs new file mode 100644 index 0000000..76943f9 --- /dev/null +++ b/src/GpgSigner.cs @@ -0,0 +1,52 @@ +using System; +using System.IO; +using System.Threading.Tasks; +using System.Collections.Generic; + +using VNLib.Tools.Build.Executor.Constants; + +namespace VNLib.Tools.Build.Executor +{ + public sealed class GpgSigner(bool enabled, string? defaultKey) + { + public bool IsEnabled { get; } = enabled; + + public async Task SignFileAsync(FileInfo file) + { + if (!IsEnabled) + { + return; + } + + List<string> args = [ "--detach-sign" ]; + + if (!string.IsNullOrWhiteSpace(defaultKey)) + { + //Set the preferred key + args.Add("--default-key"); + args.Add(defaultKey); + } + + //Add input file + args.Add(file.FullName); + + //Delete an original file + string sigFile = $"{file.FullName}.sig"; + if (File.Exists(sigFile)) + { + File.Delete(sigFile); + } + + int result = await Utils.RunProcessAsync("gpg", null, args.ToArray()); + + switch (result) + { + case 2: + case 0: + break; + default: + throw new Exception($"Failed to sign file {file.FullName}"); + } + } + } +}
\ No newline at end of file diff --git a/src/MinioUploadManager.cs b/src/MinioUploadManager.cs new file mode 100644 index 0000000..8337438 --- /dev/null +++ b/src/MinioUploadManager.cs @@ -0,0 +1,60 @@ +using System.Diagnostics.CodeAnalysis; +using System.Threading.Tasks; + +using VNLib.Tools.Build.Executor.Model; + +using static VNLib.Tools.Build.Executor.Constants.Utils; + +namespace VNLib.Tools.Build.Executor +{ + internal class MinioUploadManager : IUploadManager + { + private readonly string _minioPath; + + private MinioUploadManager(string minioPath) + { + _minioPath = minioPath; + } + + public Task CleanAllAsync(string path) + { + throw new System.NotImplementedException(); + } + + public Task DeleteFileAsync(string filePath) + { + throw new System.NotImplementedException(); + } + + public async Task UploadDirectoryAsync(string path) + { + //Recursivley copy all files in the working directory + string[] args = + { + "cp", + "--recursive", + ".", + _minioPath + }; + + //Set working dir to the supplied dir path, and run the command + int result = await RunProcessAsync("mc", path, args); + + if(result != 0) + { + throw new BuildStepFailedException($"Failed to upload directory {path} with status code {result:x}"); + } + } + + public Task UploadFileAsync(string filePath) + { + throw new System.NotImplementedException(); + } + + [return:NotNullIfNotNull(nameof(uploadPath))] + public static IUploadManager? Create(string? uploadPath) + { + return string.IsNullOrWhiteSpace(uploadPath) ? null : new MinioUploadManager(uploadPath); + } + } +}
\ No newline at end of file diff --git a/src/Model/IArtifact.cs b/src/Model/IArtifact.cs new file mode 100644 index 0000000..c3c215a --- /dev/null +++ b/src/Model/IArtifact.cs @@ -0,0 +1,23 @@ +using System; +using System.Threading.Tasks; + +namespace VNLib.Tools.Build.Executor.Model +{ + internal interface IArtifact : IDisposable + { + /// <summary> + /// Invoked when the executor requests all created artifacts load async assets + /// and update state accordingly + /// </summary> + /// <param name="vars">The taskfile variable container</param> + /// <returns>A task that completes when all assets are loaded</returns> + Task LoadAsync(TaskfileVars vars); + + /// <summary> + /// Invoked when the executor requests all artifacts cleanup assets that + /// may have been generated during a build process + /// </summary> + /// <returns>A task that completes when all assest are cleaned</returns> + Task CleanAsync(); + } +}
\ No newline at end of file diff --git a/src/Model/IBuildable.cs b/src/Model/IBuildable.cs new file mode 100644 index 0000000..dd32e03 --- /dev/null +++ b/src/Model/IBuildable.cs @@ -0,0 +1,19 @@ +using System.Threading.Tasks; + +namespace VNLib.Tools.Build.Executor.Model +{ + internal interface IBuildable : IArtifact + { + Task DoStepSyncSource(); + + Task<bool> CheckForChangesAsync(); + + Task DoStepBuild(); + + Task DoStepPostBuild(bool success); + + Task DoStepPublish(); + + Task DoRunTests(bool failOnError); + } +}
\ No newline at end of file diff --git a/src/Model/IDirectoryIndex.cs b/src/Model/IDirectoryIndex.cs new file mode 100644 index 0000000..4a30344 --- /dev/null +++ b/src/Model/IDirectoryIndex.cs @@ -0,0 +1,37 @@ +using System.IO; + +namespace VNLib.Tools.Build.Executor.Model +{ + public interface IDirectoryIndex + { + /// <summary> + /// The current build base directory + /// </summary> + DirectoryInfo BaseDir { get; } + + /// <summary> + /// The top level internal build directory + /// </summary> + DirectoryInfo BuildDir { get; } + + /// <summary> + /// The directory where log files are stored + /// </summary> + DirectoryInfo LogDir { get; } + + /// <summary> + /// Gets the build scratch directory + /// </summary> + DirectoryInfo ScratchDir { get; } + + /// <summary> + /// Gets the build checksum directory, used to store source file sums + /// </summary> + DirectoryInfo SumDir { get; } + + /// <summary> + /// The build output directory + /// </summary> + DirectoryInfo OutputDir { get; } + } +}
\ No newline at end of file diff --git a/src/Model/IFeedManager.cs b/src/Model/IFeedManager.cs new file mode 100644 index 0000000..7f56449 --- /dev/null +++ b/src/Model/IFeedManager.cs @@ -0,0 +1,16 @@ +namespace VNLib.Tools.Build.Executor.Model +{ + public interface IFeedManager + { + /// <summary> + /// Adds taskfile variables for the feed manager + /// </summary> + /// <param name="vars">The taskfile variable container</param> + void AddVariables(TaskfileVars vars); + + /// <summary> + /// The output directory of the feed + /// </summary> + string FeedOutputDir { get; } + } +}
\ No newline at end of file diff --git a/src/Model/IModuleData.cs b/src/Model/IModuleData.cs new file mode 100644 index 0000000..c41a76b --- /dev/null +++ b/src/Model/IModuleData.cs @@ -0,0 +1,19 @@ +using System.Collections.Generic; + +using LibGit2Sharp; + +namespace VNLib.Tools.Build.Executor.Model +{ + public interface IModuleData + { + ICollection<IProject> Projects { get; } + + string ModuleName { get; } + + Repository Repository { get; } + + TaskfileVars TaskVars { get; } + + IModuleFileManager FileManager { get; } + } +}
\ No newline at end of file diff --git a/src/Model/IModuleFileManager.cs b/src/Model/IModuleFileManager.cs new file mode 100644 index 0000000..cf1a325 --- /dev/null +++ b/src/Model/IModuleFileManager.cs @@ -0,0 +1,62 @@ +using System.IO; +using System.Threading.Tasks; + +namespace VNLib.Tools.Build.Executor.Model +{ + public enum ModuleFileType + { + None, + Catalog, + GitHistory, + Checksum, + LatestHash, + VersionHistory, + Archive + } + + public interface IModuleFileManager + { + /// <summary> + /// Writes the file to the module output directory + /// </summary> + /// <param name="type">The <see cref="ModuleFileType"/> to write</param> + /// <param name="fileData">The file data to write</param> + /// <returns>A task that resolves when the file has been written</returns> + Task<FileInfo> WriteFileAsync(ModuleFileType type, byte[] fileData); + + /// <summary> + /// Writes the checksum file to the sum's output directory for the given project + /// </summary> + /// <param name="project">The project to write the sum file data for</param> + /// <param name="fileData"></param> + /// <returns></returns> + Task WriteChecksumAsync(IProject project, byte[] fileData); + + /// <summary> + /// Attemts to read the checksum file data for the given project + /// </summary> + /// <param name="project">The project to get the sum data for</param> + /// <returns>The file contents of the sum file, or null if the file does not exist</returns> + Task<byte[]?> ReadCheckSumAsync(IProject project); + + /// <summary> + /// The module's output directory + /// </summary> + string OutputDir { get; } + + /// <summary> + /// Copies the given file to the project's output directory + /// </summary> + /// <param name="project"></param> + /// <param name="file"></param> + /// <returns></returns> + Task CopyArtifactToOutputAsync(IProject project, FileInfo file); + + /// <summary> + /// Gets the output directory for the given project + /// </summary> + /// <param name="project">The project to get the artifact output of</param> + /// <returns>A <see cref="DirectoryInfo"/> object describing the output dir</returns> + DirectoryInfo GetArtifactOutputDir(IProject project); + } +}
\ No newline at end of file diff --git a/src/Model/IProject.cs b/src/Model/IProject.cs new file mode 100644 index 0000000..e7f15bb --- /dev/null +++ b/src/Model/IProject.cs @@ -0,0 +1,37 @@ +using System.IO; +using System.Threading.Tasks; + +namespace VNLib.Tools.Build.Executor.Model +{ + public interface IProject : ITaskfileScope + { + /// <summary> + /// Gets the the project file + /// </summary> + FileInfo ProjectFile { get; } + + /// <summary> + /// Gets the actual project name + /// </summary> + string ProjectName { get; } + + /// <summary> + /// The msbuild project dom + /// </summary> + IProjectData ProjectData { get; } + + /// <summary> + /// A value that indicates (after a source sync) that the project + /// is considered up to date. + /// </summary> + bool UpToDate { get; set; } + + /// <summary> + /// Invoked when the executor requests all created artifacts load async assets + /// and update state accordingly + /// </summary> + /// <param name="vars">The taskfile variable container</param> + /// <returns>A task that completes when all assets are loaded</returns> + Task LoadAsync(TaskfileVars vars); + } +}
\ No newline at end of file diff --git a/src/Model/IProjectData.cs b/src/Model/IProjectData.cs new file mode 100644 index 0000000..1b4190d --- /dev/null +++ b/src/Model/IProjectData.cs @@ -0,0 +1,19 @@ +using System.IO; + + +namespace VNLib.Tools.Build.Executor.Model +{ + public interface IProjectData + { + string? Description { get; } + string? Authors { get; } + string? Copyright { get; } + string? VersionString { get; } + string? CompanyName { get; } + string? Product { get; } + string? RepoUrl { get; } + string? this[string index] { get; } + void Load(Stream stream); + string[] GetProjectRefs(); + } +}
\ No newline at end of file diff --git a/src/Model/IProjectExplorer.cs b/src/Model/IProjectExplorer.cs new file mode 100644 index 0000000..3c9c61b --- /dev/null +++ b/src/Model/IProjectExplorer.cs @@ -0,0 +1,16 @@ +using System.Collections.Generic; + +namespace VNLib.Tools.Build.Executor.Model +{ + /// <summary> + /// Represents a project explorer, capable of discovering projects within a module + /// </summary> + internal interface IProjectExplorer + { + /// <summary> + /// Discovers all projects within the module + /// </summary> + /// <returns>An enumeration of projects discovered</returns> + IEnumerable<IProject> DiscoverProjects(); + } +}
\ No newline at end of file diff --git a/src/Model/ITaskfileScope.cs b/src/Model/ITaskfileScope.cs new file mode 100644 index 0000000..6a416fa --- /dev/null +++ b/src/Model/ITaskfileScope.cs @@ -0,0 +1,22 @@ +using System.IO; + +namespace VNLib.Tools.Build.Executor.Model +{ + public interface ITaskfileScope + { + /// <summary> + /// The taskfile working directory + /// </summary> + DirectoryInfo WorkingDir { get; } + + /// <summary> + /// The taskfile variable container + /// </summary> + TaskfileVars TaskVars { get; } + + /// <summary> + /// The optional taskfile name + /// </summary> + string? TaskfileName { get; } + } +}
\ No newline at end of file diff --git a/src/Model/IUploadManager.cs b/src/Model/IUploadManager.cs new file mode 100644 index 0000000..01d7081 --- /dev/null +++ b/src/Model/IUploadManager.cs @@ -0,0 +1,15 @@ +using System.Threading.Tasks; + +namespace VNLib.Tools.Build.Executor.Model +{ + public interface IUploadManager + { + Task CleanAllAsync(string path); + + Task DeleteFileAsync(string filePath); + + Task UploadDirectoryAsync(string path); + + Task UploadFileAsync(string filePath); + } +}
\ No newline at end of file diff --git a/src/Model/TaskfileVars.cs b/src/Model/TaskfileVars.cs new file mode 100644 index 0000000..c0e6cdc --- /dev/null +++ b/src/Model/TaskfileVars.cs @@ -0,0 +1,48 @@ +using System; +using System.Collections.Generic; + +namespace VNLib.Tools.Build.Executor.Model +{ + /// <summary> + /// Represents a collection of taskfile "environment" variables + /// </summary> + public sealed class TaskfileVars + { + private readonly Dictionary<string, string> vars; + + public TaskfileVars() + { + vars = new(StringComparer.OrdinalIgnoreCase); + } + + private TaskfileVars(IEnumerable<KeyValuePair<string, string>> values) + { + vars = new(values, StringComparer.OrdinalIgnoreCase); + } + + /// <summary> + /// Gets all variables as a readonly dictionary + /// </summary> + /// <returns>The collection of environment variables</returns> + public IReadOnlyDictionary<string, string> GetVariables() => vars; + + /// <summary> + /// Sets a taskfile environment variable + /// </summary> + /// <param name="key">The variable name</param> + /// <param name="value">The optional variable value</param> + public void Set(string key, string? value) => vars[key] = value ?? string.Empty; + + /// <summary> + /// Removes a taskfile environment variable + /// </summary> + /// <param name="key">The name of the variable to remove</param> + public void Remove(string key) => vars.Remove(key); + + /// <summary> + /// Clones the current taskfile variables into an independent instance + /// </summary> + /// <returns>The new <see cref="TaskfileVars"/> instance</returns> + public TaskfileVars Clone() => new (vars); + } +}
\ No newline at end of file diff --git a/src/Modules/GitCodeModule.cs b/src/Modules/GitCodeModule.cs new file mode 100644 index 0000000..630922d --- /dev/null +++ b/src/Modules/GitCodeModule.cs @@ -0,0 +1,42 @@ +using System.IO; +using System.Linq; +using System.Threading.Tasks; + +using VNLib.Tools.Build.Executor.Constants; +using VNLib.Tools.Build.Executor.Model; + +namespace VNLib.Tools.Build.Executor.Modules +{ + internal sealed class GitCodeModule : ModuleBase + { + private string _moduleName; + + public override IProjectExplorer ModuleExplorer { get; } + + public override string ModuleName => _moduleName; + + public GitCodeModule(BuildConfig config, DirectoryInfo root) + : base(config, root) + { + ModuleExplorer = new MsBuildModuleExplorer(config, this, root); + _moduleName = root.Name; //Default to dir name + } + + ///<inheritdoc/> + public override Task LoadAsync(TaskfileVars vars) + { + //Try to load a solution file in the top-level module directory + FileInfo? sln = WorkingDir.EnumerateFiles("*.sln", SearchOption.TopDirectoryOnly).FirstOrDefault(); + + if(sln is not null) + { + //Remove the .build extension from the solution file name for proper module name + _moduleName = Path.GetFileNameWithoutExtension(sln.Name).Replace(".build", string.Empty); + + vars.Set("SOLUTION_FILE_NAME", sln.Name); + } + + return base.LoadAsync(vars); + } + } +}
\ No newline at end of file diff --git a/src/Modules/ModuleBase.cs b/src/Modules/ModuleBase.cs new file mode 100644 index 0000000..750cf61 --- /dev/null +++ b/src/Modules/ModuleBase.cs @@ -0,0 +1,258 @@ +using System; +using System.IO; +using System.Linq; +using System.Threading.Tasks; +using System.Collections.Generic; + +using LibGit2Sharp; + +using VNLib.Tools.Build.Executor.Model; +using VNLib.Tools.Build.Executor.Constants; +using VNLib.Tools.Build.Executor.Extensions; +using static VNLib.Tools.Build.Executor.Constants.Config; + +namespace VNLib.Tools.Build.Executor.Modules +{ + /// <summary> + /// Represents a base class for all modules to inherit from + /// </summary> + internal abstract class ModuleBase : IArtifact, IBuildable, IModuleData, ITaskfileScope + { + protected readonly BuildConfig Config; + protected readonly TaskFile TaskFile; + + ///<inheritdoc/> + public TaskfileVars TaskVars { get; private set; } + + ///<inheritdoc/> + public DirectoryInfo WorkingDir { get; } + + ///<inheritdoc/> + public abstract string ModuleName { get; } + + ///<inheritdoc/> + public ICollection<IProject> Projects { get; } = new LinkedList<IProject>(); + + ///<inheritdoc/> + public IModuleFileManager FileManager { get; } + + ///<inheritdoc/> + public string? TaskfileName { get; protected set; } + + /// <summary> + /// The git repository of the module + /// </summary> + public Repository Repository { get; } + + /// <summary> + /// The project explorer for the module + /// </summary> + public abstract IProjectExplorer ModuleExplorer { get; } + + public ModuleBase(BuildConfig config, DirectoryInfo root) + { + WorkingDir = root; + Config = config; + + //Default to module taskfile name + TaskfileName = config.ModuleTaskFileName; + + //Init repo for root working dir + Repository = new(root.FullName); + FileManager = new ModuleFileManager(config, this); + TaskVars = null!; + + TaskFile = new(config.TaskExeName, () => ModuleName); + } + + ///<inheritdoc/> + public virtual async Task LoadAsync(TaskfileVars vars) + { + if(Repository?.Head?.Tip?.Sha is null) + { + throw new BuildStepFailedException("This repository does not have any commit history. Cannot continue"); + } + + //Store paraent vars + TaskVars = vars; + + string moduleSemVer = this.GetModuleCiVersion(Config.DefaultCiVersion, Config.SemverStyle); + + //Build module local environment variables + TaskVars.Set("MODULE_NAME", ModuleName); + TaskVars.Set("OUTPUT_DIR", FileManager.OutputDir); + TaskVars.Set("MODULE_DIR", WorkingDir.FullName); + + //Store current head-sha before update step + TaskVars.Set("HEAD_SHA", Repository.Head.Tip.Sha); + TaskVars.Set("BRANCH_NAME", Repository.Head.FriendlyName); + TaskVars.Set("BUILD_VERSION", moduleSemVer); + + //Full path to module archive file + TaskVars.Set("FULL_ARCHIVE_FILE_NAME", Path.Combine(WorkingDir.FullName, Config.SourceArchiveName)); + TaskVars.Set("ARCHIVE_FILE_NAME", Config.SourceArchiveName); + TaskVars.Set("ARCHIVE_FILE_FORMAT", Config.SourceArchiveFormat); + + //Remove any previous projects + Projects.Clear(); + + Log.Information("Discovering projects in module {sln}", ModuleName); + + //Discover all projects in for the module + foreach (IProject project in ModuleExplorer.DiscoverProjects()) + { + //Store in collection + Projects.Add(project); + } + + //Load all projects + await Projects.RunAllAsync(p => p.LoadAsync(TaskVars.Clone())); + + Log.Information("Sucessfully loaded {count} projects into module {sln}", Projects.Count, ModuleName); + Log.Information("{modname} CI build SemVer will be {semver}", ModuleName, moduleSemVer); + } + + ///<inheritdoc/> + public virtual async Task DoStepSyncSource() + { + Log.Information("Checking for source code updates in module {mod}", ModuleName); + + //Do a git pull to update our sources + await TaskFile.ExecCommandAsync(this, TaskfileComamnd.Update, true); + + //Set the latest commit sha after an update + TaskVars.Set("HEAD_SHA", Repository.Head.Tip.Sha); + + //Update lastest build number + TaskVars.Set("BUILD_VERSION", this.GetModuleCiVersion(Config.DefaultCiVersion, Config.SemverStyle)); + + //Update module semver after source sync + string moduleSemVer = this.GetModuleCiVersion(Config.DefaultCiVersion, Config.SemverStyle); + TaskVars.Set("BUILD_VERSION", moduleSemVer); + Log.Information("{modname} CI build SemVer will now be {semver}", ModuleName, moduleSemVer); + } + + ///<inheritdoc/> + public virtual async Task<bool> CheckForChangesAsync() + { + //Check source for updates + await Projects.RunAllAsync(p => FileManager.CheckSourceChangedAsync(p, Config, Repository.Head.Tip.Sha)); + + //Check if any project is not up-to-date + return Projects.Any(static p => !p.UpToDate); + } + + ///<inheritdoc/> + public virtual async Task DoStepBuild() + { + //Clean the output dir + FileManager.CleanOutput(); + //Recreate the output dir + FileManager.CreateOutput(); + + //Run taskfile to build + await TaskFile.ExecCommandAsync(this, TaskfileComamnd.Build, true); + + //Run build for all projects + foreach (IProject proj in Projects) + { + //Exec + await TaskFile.ExecCommandAsync(proj, TaskfileComamnd.Build, true); + } + } + + ///<inheritdoc/> + public virtual async Task DoStepPostBuild(bool success) + { + //Run taskfile postbuild, not required to produce a sucessful result + await TaskFile.ExecCommandAsync(this, success ? TaskfileComamnd.PostbuildSuccess : TaskfileComamnd.PostbuildFailure, false); + + //Run postbuild for all projects + foreach (IProject proj in Projects) + { + //Run postbuild for projects + await TaskFile.ExecCommandAsync(proj, success ? TaskfileComamnd.PostbuildSuccess : TaskfileComamnd.PostbuildFailure, false); + } + + //Run postbuild for all projects + await Projects.RunAllAsync(async (p) => + { + //If the operation was a success, commit the sum change + if (success) + { + Log.Verbose("Committing sum change for {sm}", p.ProjectName); + //Commit sum changes now that build has completed successfully + await FileManager.CommitSumChangeAsync(p); + } + }); + } + + ///<inheritdoc/> + public virtual async Task DoStepPublish() + { + //Run taskfile postbuild, not required to produce a sucessful result + await TaskFile.ExecCommandAsync(this, TaskfileComamnd.Publish, true); + + //Run postbuild for all projects + foreach (IProject proj in Projects) + { + //Run postbuild for projects + await TaskFile.ExecCommandAsync(proj, TaskfileComamnd.Publish, true); + } + } + + ///<inheritdoc/> + public virtual async Task DoRunTests(bool failOnError) + { + //Run taskfile to build + await TaskFile.ExecCommandAsync(this, TaskfileComamnd.Test, failOnError); + + //Run build for all projects + foreach (IProject proj in Projects) + { + //Exec + await TaskFile.ExecCommandAsync(proj, TaskfileComamnd.Test, failOnError); + } + } + + ///<inheritdoc/> + public virtual async Task CleanAsync() + { + try + { + //Run taskfile to build + await TaskFile.ExecCommandAsync(this, TaskfileComamnd.Clean, true); + + //Clean all projects + foreach (IProject proj in Projects) + { + //Clean the project output dir + await TaskFile.ExecCommandAsync(proj, TaskfileComamnd.Clean, true); + } + + //Clean the output dir + FileManager.CleanOutput(); + } + catch (BuildStepFailedException) + { + throw; + } + catch (Exception ex) + { + throw new BuildStepFailedException("Failed to remove the module output directory", ex, ModuleName); + } + } + + public override string ToString() => ModuleName; + + + public virtual void Dispose() + { + //Dispose the respository + Repository.Dispose(); + + //empty list + Projects.Clear(); + } + } +}
\ No newline at end of file diff --git a/src/Modules/ModuleFileManager.cs b/src/Modules/ModuleFileManager.cs new file mode 100644 index 0000000..4d6ed69 --- /dev/null +++ b/src/Modules/ModuleFileManager.cs @@ -0,0 +1,92 @@ +using System; +using System.IO; +using System.Threading.Tasks; + +using VNLib.Tools.Build.Executor.Model; +using VNLib.Tools.Build.Executor.Extensions; +using VNLib.Tools.Build.Executor.Constants; + +namespace VNLib.Tools.Build.Executor.Modules +{ + + public sealed class ModuleFileManager(BuildConfig config, IModuleData ModData) : IModuleFileManager + { + private readonly IDirectoryIndex Index = config.Index; + + ///<inheritdoc/> + public string OutputDir => Path.Combine(Index.OutputDir.FullName, ModData.ModuleName); + + ///<inheritdoc/> + public async Task CopyArtifactToOutputAsync(IProject project, FileInfo file) + { + string targetDir = GetProjectTargetDir(project); + + //Project artifacts are versioned by the latest git commit hash + string outputFile = Path.Combine(targetDir, file.Name); + + //Create the target directory if it doesn't exist + Directory.CreateDirectory(targetDir); + + //Copy the file to the output directory + FileInfo output = file.CopyTo(outputFile, true); + + //Compute the file hash of the new output file + await output.ComputeFileHashAsync(config.HashFuncName); + } + + ///<inheritdoc/> + public DirectoryInfo GetArtifactOutputDir(IProject project) + { + string path = GetProjectTargetDir(project); + return new DirectoryInfo(path); + } + + ///<inheritdoc/> + public Task<byte[]?> ReadCheckSumAsync(IProject project) + { + string sumFile = Path.Combine(Index.SumDir.FullName, $"{ModData.ModuleName}-{project.GetSafeProjectName()}.json"); + return File.Exists(sumFile) ? File.ReadAllBytesAsync(sumFile) : Task.FromResult<byte[]?>(null); + } + + ///<inheritdoc/> + public Task WriteChecksumAsync(IProject project, byte[] fileData) + { + //Create sum file inside the sum directory + string sumFile = Path.Combine(Index.SumDir.FullName, $"{ModData.ModuleName}-{project.GetSafeProjectName()}.json"); + return File.WriteAllBytesAsync(sumFile, fileData); + } + + ///<inheritdoc/> + public async Task<FileInfo> WriteFileAsync(ModuleFileType type, byte[] fileData) + { + //Get the file path for the given type + string filePath = type switch + { + //Catalog is written to the version pointed to by the latest git commit hash + ModuleFileType.Catalog => $"{OutputDir}/{GetLatestTagOrSha()}/index.json", + ModuleFileType.GitHistory => $"{OutputDir}/git.json", + ModuleFileType.LatestHash => $"{OutputDir}/@latest", + ModuleFileType.VersionHistory => $"{OutputDir}/versions.json", + //Store project archive + ModuleFileType.Archive => $"{OutputDir}/{GetLatestTagOrSha()}/archive.tgz", + _ => throw new ArgumentOutOfRangeException(nameof(type), type, null), + }; + + await File.WriteAllBytesAsync($"{filePath}", fileData); + //Return new file handle + return new FileInfo(filePath); + } + + private string GetProjectTargetDir(IProject project) + { + //get last tag + return Path.Combine(OutputDir, GetLatestTagOrSha(), project.GetSafeProjectName()); + } + + private string GetLatestTagOrSha() + { + return ModData.Repository.Head.Tip.Sha; + } + + } +}
\ No newline at end of file diff --git a/src/Modules/MsBuildModuleExplorer.cs b/src/Modules/MsBuildModuleExplorer.cs new file mode 100644 index 0000000..fa4592e --- /dev/null +++ b/src/Modules/MsBuildModuleExplorer.cs @@ -0,0 +1,89 @@ +using System; +using System.IO; +using System.Linq; +using System.Collections.Generic; + +using Microsoft.Build.Construction; + +using VNLib.Tools.Build.Executor.Model; +using VNLib.Tools.Build.Executor.Projects; +using VNLib.Tools.Build.Executor.Constants; +using static VNLib.Tools.Build.Executor.Constants.Config; +using VNLib.Tools.Build.Executor.Extensions; + +namespace VNLib.Tools.Build.Executor.Modules +{ + + /* + * Discovers all projects within a dotnet solution file. First it finds the solution file in the + * working directory, then it parses the solution file to find all projects within the solution. + * + * Leaf projects are then discovered by searching for all project files within the working directory + * that are not part of the solution. + */ + + internal sealed class MsBuildModuleExplorer(BuildConfig config, IModuleData Module, DirectoryInfo ModuleDir) : IProjectExplorer + { + ///<inheritdoc/> + public IEnumerable<IProject> DiscoverProjects() + { + LinkedList<IProject> projects = new(); + + //First load the solution file + FileInfo? slnFile = ModuleDir.EnumerateFiles("*.sln", SearchOption.TopDirectoryOnly).FirstOrDefault(); + + if(slnFile is not null) + { + GetProjectsForSoution(slnFile, projects); + } + + //Capture any leaf projects that are not part of the solution + IEnumerable<FileInfo> leafProjects = ModuleDir.EnumerateFiles("package.json", SearchOption.AllDirectories).Distinct(); + + //Capture them + foreach (FileInfo leafProjFile in leafProjects) + { + //Create relative file path + string realtivePath = leafProjFile.FullName.Replace(ModuleDir.FullName, string.Empty).TrimStart(Path.DirectorySeparatorChar); + + //If the leaf project is ignored, skip it + if (Module.Repository.IsFileIgnored(realtivePath)) + { + continue; + } + + //Create the leaf project + LeafProject project = new(config, leafProjFile); + + Log.Verbose("Discovered leaf project in {proj} ", leafProjFile.DirectoryName); + + projects.AddLast(project); + } + + return projects; + } + + private static void GetProjectsForSoution(FileInfo slnFile, LinkedList<IProject> projects) + { + //Parse solution + SolutionFile Solution = SolutionFile.Parse(slnFile.FullName); + + //Loop through all artificats within the solution + foreach (ProjectInSolution proj in Solution.ProjectsInOrder.Where(static p => p.ProjectType == SolutionProjectType.KnownToBeMSBuildFormat)) + { + //Ignore test projects in a solution + if(proj.ProjectName.Contains("test", StringComparison.OrdinalIgnoreCase)) + { + continue; + } + + //Create the new project artifact + DotnetProject project = new(new(proj.AbsolutePath), proj.ProjectName); + + Log.Verbose("Discovered project {proj} ", proj.ProjectName); + + projects.AddLast(project); + } + } + } +}
\ No newline at end of file diff --git a/src/Program.cs b/src/Program.cs new file mode 100644 index 0000000..1a9ad06 --- /dev/null +++ b/src/Program.cs @@ -0,0 +1,38 @@ +using Typin; +using Typin.Console; + +using VNLib.Tools.Build.Executor.Constants; + +using Microsoft.Extensions.DependencyInjection; + + +namespace VNLib.Tools.Build.Executor +{ + + sealed class Program + { + static int Main(string[] argsv) + { + return new CliApplicationBuilder() + .AddCommandsFromThisAssembly() + .UseConsole<SystemConsole>() + .UseTitle("VNBuild Copyright (c) Vaughn Nugent") + .UseStartupMessage("VNBuild Copyright (c) Vaughn Nugent") + .UseVersionText("0.1.0") + .ConfigureServices(services => + { + //Init new pipeline and add to service collection + services + .AddSingleton(Config.Log) + .AddSingleton<BuildPipeline>() + .AddSingleton(new ConfigManager(Semver.SemVersionStyles.Any)); + + }) + .Build() + .RunAsync() + .AsTask() + .GetAwaiter() + .GetResult(); + } + } +}
\ No newline at end of file diff --git a/src/Projects/DotnetProject.cs b/src/Projects/DotnetProject.cs new file mode 100644 index 0000000..5ee1ed0 --- /dev/null +++ b/src/Projects/DotnetProject.cs @@ -0,0 +1,38 @@ + +using System; +using System.IO; +using System.Threading.Tasks; + +using VNLib.Tools.Build.Executor.Model; + +namespace VNLib.Tools.Build.Executor.Projects +{ + + internal sealed class DotnetProject : ModuleProject + { + public override IProjectData ProjectData { get; } + + + public DotnetProject(FileInfo projectFile, string projectName):base(projectFile, projectName) + { + //Create the porject dom + ProjectData = new DotnetProjectDom(); + } + + ///<inheritdoc/> + public override async Task LoadAsync(TaskfileVars vars) + { + //Load project dom + await base.LoadAsync(vars); + + //Set .NET specific vars + TaskVars.Set("TARGET_FRAMEWORK", ProjectData["TargetFramework"] ?? string.Empty); + TaskVars.Set("PROJ_ASM_NAME", ProjectData["AssemblyName"] ?? string.Empty); + } + + public override string ToString() => ProjectName; + + public override void Dispose() + { } + } +}
\ No newline at end of file diff --git a/src/Projects/DotnetProjectDom.cs b/src/Projects/DotnetProjectDom.cs new file mode 100644 index 0000000..9b44747 --- /dev/null +++ b/src/Projects/DotnetProjectDom.cs @@ -0,0 +1,60 @@ + +using System; +using System.IO; +using System.Xml; +using System.Collections.Generic; + +using VNLib.Tools.Build.Executor.Model; + +namespace VNLib.Tools.Build.Executor.Projects +{ + + internal sealed class DotnetProjectDom : IProjectData + { + private readonly XmlDocument _dom; + + public DotnetProjectDom() + { + _dom = new(); + } + + public void Load(Stream stream) + { + _dom.Load(stream); + } + + private XmlNode? project => _dom["Project"]; + + public string? Description => GetProperty("Description"); + public string? Authors => GetProperty("Authors"); + public string? Copyright => GetProperty("Copyright"); + public string? VersionString => GetProperty("Version"); + public string? CompanyName => GetProperty("Company"); + public string? Product => GetProperty("Product"); + public string? RepoUrl => GetProperty("RepositoryUrl"); + + public string? this[string index] => GetProperty(index); + + public string? GetProperty(string name) => GetItemAtPath($"PropertyGroup/{name}"); + + public string? GetItemAtPath(string path) => project!.SelectSingleNode(path)?.InnerText; + + public string[] GetProjectRefs() + { + //Get the item group attr + XmlNodeList? projectRefs = project?.SelectNodes("ItemGroup/ProjectReference"); + + if (projectRefs != null) + { + List<string> refs = new(); + foreach (XmlNode projRef in projectRefs) + { + //Get the project ref via its include attribute + refs.Add(projRef.Attributes["Include"].Value!); + } + return refs.ToArray(); + } + return Array.Empty<string>(); + } + } +}
\ No newline at end of file diff --git a/src/Projects/LeafProject.cs b/src/Projects/LeafProject.cs new file mode 100644 index 0000000..9b155cf --- /dev/null +++ b/src/Projects/LeafProject.cs @@ -0,0 +1,37 @@ +using System.IO; +using System.Threading.Tasks; + +using VNLib.Tools.Build.Executor.Constants; +using VNLib.Tools.Build.Executor.Model; + +namespace VNLib.Tools.Build.Executor.Projects +{ + internal sealed class LeafProject(BuildConfig config, FileInfo projectFile) : ModuleProject(projectFile) + { + ///<inheritdoc/> + public override IProjectData ProjectData { get; } = new NativeProjectDom(); + + ///<inheritdoc/> + protected override FileInfo? PackageInfoFile => new(Path.Combine(WorkingDir.FullName, "package.json")); + + public override async Task LoadAsync(TaskfileVars vars) + { + await base.LoadAsync(vars); + + //Set the project name to the product name if set, otherwise use the working dir name + ProjectName = ProjectData.Product ?? WorkingDir.Name; + + //Get the binary dir from the project file, or use the default + string? binaryDir = ProjectData["output_dir"] ?? ProjectData["output"] ?? config.ProjectBinDir; + + //Overide the project name from the pacakge file if set + TaskVars.Set("PROJECT_NAME", ProjectName); + TaskVars.Set("BINARY_DIR", binaryDir); + } + + public override string ToString() => ProjectName; + + public override void Dispose() + { } + } +}
\ No newline at end of file diff --git a/src/Projects/ModuleProject.cs b/src/Projects/ModuleProject.cs new file mode 100644 index 0000000..4643aa0 --- /dev/null +++ b/src/Projects/ModuleProject.cs @@ -0,0 +1,118 @@ +using System; +using System.IO; +using System.Threading.Tasks; + +using VNLib.Tools.Build.Executor.Model; +using VNLib.Tools.Build.Executor.Extensions; + +namespace VNLib.Tools.Build.Executor.Projects +{ + internal abstract class ModuleProject : IProject + { + ///<inheritdoc/> + public FileInfo ProjectFile { get; } + + ///<inheritdoc/> + public string ProjectName { get; protected set; } + + ///<inheritdoc/> + public abstract IProjectData ProjectData { get; } + + ///<inheritdoc/> + public bool UpToDate { get; set; } + + ///<inheritdoc/> + public DirectoryInfo WorkingDir { get; protected set; } + + ///<inheritdoc/> + public TaskfileVars TaskVars { get; protected set; } + + ///<inheritdoc/> + public string? TaskfileName { get; protected set; } + + /// <summary> + /// Gets the package info file for the project + /// </summary> + protected virtual FileInfo? PackageInfoFile { get; } + + public ModuleProject(FileInfo projectFile, string? projectName = null) + { + ProjectFile = projectFile; + + //Default project name to the file name + ProjectName = projectName ?? Path.GetFileNameWithoutExtension(ProjectFile.Name); + + //Default up-to-date false + UpToDate = false; + + //Default working dir to the project file's directory + WorkingDir = ProjectFile.Directory!; + + TaskVars = null!; + } + + ///<inheritdoc/> + public virtual async Task LoadAsync(TaskfileVars vars) + { + TaskVars = vars; + + await LoadProjectDom(); + + //Set some local environment variables + + //Set local environment variables + TaskVars.Set("PROJECT_NAME", ProjectName); + TaskVars.Set("PROJECT_DIR", WorkingDir.FullName); + TaskVars.Set("IS_PROJECT", bool.TrueString); + + //Store project vars + TaskVars.Set("PROJ_VERSION", ProjectData.VersionString ?? string.Empty); + TaskVars.Set("PROJ_DESCRIPTION", ProjectData.Description ?? string.Empty); + TaskVars.Set("PROJ_AUTHOR", ProjectData.Authors ?? string.Empty); + TaskVars.Set("PROJ_COPYRIGHT", ProjectData.Copyright ?? string.Empty); + TaskVars.Set("PROJ_COMPANY", ProjectData.CompanyName ?? string.Empty); + TaskVars.Set("RPOJ_URL", ProjectData.RepoUrl ?? string.Empty); + + TaskVars.Set("SAFE_PROJ_NAME", this.GetSafeProjectName()); + } + + /// <summary> + /// Loads the project's XML dom from its msbuild project file + /// </summary> + /// <returns>A task that resolves when the dom is built</returns> + public async Task LoadProjectDom() + { + using MemoryStream ms = new(); + + FileInfo dom = ProjectFile; + + if (PackageInfoFile?.Exists == true) + { + dom = PackageInfoFile; + } + + try + { + //Get the project file + await using (FileStream projData = dom.OpenRead()) + { + await projData.CopyToAsync(ms); + } + + //reset stream + ms.Seek(0, SeekOrigin.Begin); + + //Load the project dom + ProjectData.Load(ms); + } + catch (Exception ex) + { + throw new Exception($"Failed to load project dom file {dom.FullName}", ex); + } + } + + public abstract void Dispose(); + + public override string ToString() => ProjectName; + } +}
\ No newline at end of file diff --git a/src/Projects/NativeProjectDom.cs b/src/Projects/NativeProjectDom.cs new file mode 100644 index 0000000..f23a4a5 --- /dev/null +++ b/src/Projects/NativeProjectDom.cs @@ -0,0 +1,57 @@ +using System; +using System.IO; +using System.Text.Json; +using System.Collections.Generic; + +using VNLib.Tools.Build.Executor.Model; + +namespace VNLib.Tools.Build.Executor.Projects +{ + internal sealed class NativeProjectDom : IProjectData + { + private Dictionary<string, string> _properties; + + internal NativeProjectDom() + { + _properties = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase); + } + + public string? this[string index] => _properties.GetValueOrDefault(index); + + public string? Description => this["description"]; + public string? Authors => this["author"]; + public string? Copyright => this["copyright"]; + public string? VersionString => this["version"]; + public string? CompanyName => this["company"]; + public string? Product => this["name"]; + public string? RepoUrl => this["repository"]; + public string? OutputDir => this["output_dir"]; + + public string[] GetProjectRefs() + { + return Array.Empty<string>(); + } + + public void Load(Stream stream) + { + //Read the json file + using JsonDocument doc = JsonDocument.Parse(stream, new JsonDocumentOptions + { + CommentHandling = JsonCommentHandling.Skip, + AllowTrailingCommas = true, + }); + + //Clear old properties + _properties.Clear(); + + //Load new properties that are strings only + foreach (JsonProperty prop in doc.RootElement.EnumerateObject()) + { + if(prop.Value.ValueKind == JsonValueKind.String) + { + _properties[prop.Name] = prop.Value.GetString() ?? string.Empty; + } + } + } + } +}
\ No newline at end of file diff --git a/src/Properties/launchSettings.json b/src/Properties/launchSettings.json new file mode 100644 index 0000000..8701aaa --- /dev/null +++ b/src/Properties/launchSettings.json @@ -0,0 +1,9 @@ +{ + "profiles": { + "VNLib.Tools.Build.Executor": { + "commandName": "Project", + "commandLineArgs": "publish -v -c -x vnlib.browser", + "workingDirectory": "B:\\Progamming\\vnlib.browser" + } + } +}
\ No newline at end of file diff --git a/src/SleetFeedManager.cs b/src/SleetFeedManager.cs new file mode 100644 index 0000000..bc77ff1 --- /dev/null +++ b/src/SleetFeedManager.cs @@ -0,0 +1,53 @@ +using System; +using System.Diagnostics.CodeAnalysis; +using System.IO; +using System.Text.Json; + +using VNLib.Tools.Build.Executor.Model; + +namespace VNLib.Tools.Build.Executor +{ + internal sealed class SleetFeedManager : IFeedManager + { + private readonly string SleetConfigFile; + + ///<inheritdoc/> + public string FeedOutputDir { get; } + + private SleetFeedManager(string indexFilex, string outputDir) + { + //Search for the sleet file in the build dir + SleetConfigFile = indexFilex; + FeedOutputDir = outputDir; + } + + ///<inheritdoc/> + public void AddVariables(TaskfileVars vars) + { + vars.Set("SLEET_DIR", FeedOutputDir); + vars.Set("SLEET_CONFIG_PATH", SleetConfigFile); + } + + /// <summary> + /// Attempts to create a new sleet feed manager from the given directory index + /// if the index contains a sleet feed. Returns null if no sleet feed was found + /// </summary> + /// <returns>The feed manager if found, null otherwise</returns> + /// <exception cref="FileNotFoundException"></exception> + /// <exception cref="ArgumentException"></exception> + [return:NotNullIfNotNull(nameof(feedPath))] + public static IFeedManager? GetSleetFeed(string? feedPath) + { + if(string.IsNullOrWhiteSpace(feedPath)) + { + return null; + } + + //Read the sleet config file + byte[] sleetIndexFile = File.ReadAllBytes(feedPath); + using JsonDocument doc = JsonDocument.Parse(sleetIndexFile); + string rootDir = doc.RootElement.GetProperty("root").GetString() ?? throw new ArgumentException("The sleet output directory was not specified"); + return new SleetFeedManager(feedPath, rootDir); + } + } +}
\ No newline at end of file diff --git a/src/TaskFile.cs b/src/TaskFile.cs new file mode 100644 index 0000000..6eef779 --- /dev/null +++ b/src/TaskFile.cs @@ -0,0 +1,95 @@ +using System; +using System.IO; +using System.Threading.Tasks; +using System.Collections.Generic; + +using VNLib.Tools.Build.Executor.Model; +using VNLib.Tools.Build.Executor.Constants; +using static VNLib.Tools.Build.Executor.Constants.Utils; + +namespace VNLib.Tools.Build.Executor +{ + public enum TaskfileComamnd + { + Clean, + Build, + Upload, + Update, + PostbuildSuccess, + PostbuildFailure, + Publish, + Test, + } + + /// <summary> + /// Represents a controller for the TaskFile build system + /// </summary> + public sealed class TaskFile(string taskFilePath, Func<string> moduleName) + { + /// <summary> + /// Executes the desired Taskfile command with the given user args for + /// the configured manager. + /// </summary> + /// <param name="command">The command to execute</param> + /// <param name="userArgs">Additional user arguments to pass to Task </param> + /// <returns>A task that completes with the status code of the operation</returns> + public async Task ExecCommandAsync(ITaskfileScope scope, TaskfileComamnd command, bool throwIfFailed) + { + //Get working copy of vars + IReadOnlyDictionary<string, string> vars = scope.TaskVars.GetVariables(); + + //Specify taskfile if it is set + List<string> args = []; + if(!string.IsNullOrWhiteSpace(scope.TaskfileName)) + { + //If taskfile is set, we need to make sure it is in the working dir to execute it, otherwise just exit + if(!File.Exists(Path.Combine(scope.WorkingDir.FullName, scope.TaskfileName))) + { + return; + } + + args.Add("-t"); + args.Add(scope.TaskfileName); + } + + //Always add command last + args.Add(GetCommand(command)); + + //Exec task in the module dir + int result = await RunProcessAsync(taskFilePath, scope.WorkingDir.FullName, args.ToArray(), vars); + + if(throwIfFailed) + { + ThrowIfStepFailed(result, command); + } + } + + private static string GetCommand(TaskfileComamnd cmd) + { + return cmd switch + { + TaskfileComamnd.Clean => "clean", + TaskfileComamnd.Build => "build", + TaskfileComamnd.Upload => "upload", + TaskfileComamnd.Update => "update", + TaskfileComamnd.PostbuildSuccess => "postbuild_success", + TaskfileComamnd.PostbuildFailure => "postbuild_failed", + TaskfileComamnd.Publish => "publish", + TaskfileComamnd.Test => "test", + _ => throw new NotImplementedException() + }; + } + + private void ThrowIfStepFailed(int result, TaskfileComamnd cmd) + { + switch (result) + { + case 200: //Named task not found + return; + case 201: + Utils.ThrowIfStepFailed(false, $"Task failed to execute task command {cmd}", moduleName.Invoke()); + return; + } + } + } +}
\ No newline at end of file diff --git a/src/vnbuild.csproj b/src/vnbuild.csproj new file mode 100644 index 0000000..b3e32df --- /dev/null +++ b/src/vnbuild.csproj @@ -0,0 +1,45 @@ +<Project Sdk="Microsoft.NET.Sdk"> + + <PropertyGroup> + <OutputType>Exe</OutputType> + <TargetFramework>net8.0</TargetFramework> + <Nullable>enable</Nullable> + <AssemblyName>vnbuild</AssemblyName> + <RootNamespace>VNLib.Tools.Build.Executor</RootNamespace> + </PropertyGroup> + + <PropertyGroup> + <Authors>Vaughn Nugent</Authors> + <Company>Vaughn Nugent</Company> + <Description>Automatically builds and produces binaries from git-based modules with proper indexing for web based deployments</Description> + <Copyright>Copyright © 2024 Vaughn Nugent</Copyright> + <Product>vnbuild</Product> + <PackageProjectUrl>https://www.vaughnnugent.com/resources/software/modules/vnbuild</PackageProjectUrl> + <RepositoryUrl>https://github.com/VnUgE/vnbuild/</RepositoryUrl> + <PackageReadmeFile>README.md</PackageReadmeFile> + <PackageLicenseFile>LICENSE</PackageLicenseFile> + </PropertyGroup> + + <ItemGroup> + <None Include="..\LICENSE"> + <Pack>True</Pack> + <PackagePath>\</PackagePath> + <CopyToOutputDirectory>Always</CopyToOutputDirectory> + </None> + <None Include="..\README.md"> + <Pack>True</Pack> + <PackagePath>\</PackagePath> + </None> + </ItemGroup> + + <ItemGroup> + <PackageReference Include="Typin" Version="3.1.0" /> + <PackageReference Include="LibGit2Sharp" Version="0.30.0" /> + <PackageReference Include="Microsoft.Build" Version="17.10.0-preview-24081-01" /> + <PackageReference Include="Semver" Version="3.0.0-beta.1" /> + <PackageReference Include="Serilog" Version="4.0.0-dev-02166" /> + <PackageReference Include="Serilog.Sinks.Console" Version="5.1.0-dev-00943" /> + <PackageReference Include="Serilog.Sinks.File" Version="5.0.1-dev-00972" /> + </ItemGroup> + +</Project> |