using System;
using System.IO;
using System.Linq;
using System.Text;
using System.Text.Json;
using System.Threading.Tasks;
using System.Collections.Generic;
using LibGit2Sharp;
using Semver;
using VNLib.Tools.Build.Executor.Constants;
using VNLib.Tools.Build.Executor.Model;
using VNLib.Tools.Build.Executor.Extensions;
using VNLib.Tools.Build.Executor.Projects;
using static VNLib.Tools.Build.Executor.Constants.Config;
namespace VNLib.Tools.Build.Executor.Publishing
{
public sealed class BuildPublisher(BuildConfig config, GpgSigner signer)
{
public bool SignEnabled => signer.IsEnabled;
///
/// Prepares the module output and its collection of file details for publishing
/// then runs the upload step
///
///
/// A task that completes when the module's output has been created
public async Task PrepareModuleOutput(IModuleData module)
{
//Copy project artifacts to output directory
await CopyProjectOutputToModuleOutputAsync(module);
//Copy source archive
string? archiveFile = await CopySourceArchiveToOutput(module, module.FileManager);
Log.Information("Building module {mod} catalog and git history", module.ModuleName);
//Build module catalog
await BuildModuleCatalogAsync(module, config.SemverStyle, archiveFile);
Log.Information("Building module {mod} git history", module.ModuleName);
//Build git history
await BuildModuleGitHistoryAsync(module);
Log.Information("Building module {mod} version history", module.ModuleName);
//build version history
await BuildModuleVersionHistory(module);
Log.Information("Moving module {mod} artifacts to the output", module.ModuleName);
}
///
/// Uploads the modules output to the remote
///
/// The module containing the information to upload
///
public Task UploadModuleOutput(IUploadManager Uploader, IModuleData module)
{
Log.Information("Uploading module {mod}", module.ModuleName);
//Upload the entire output directory
return Uploader.UploadDirectoryAsync(module.FileManager.OutputDir);
}
/*
* Builds the project catalog file and publishes it to the module file manager
*/
private async Task BuildModuleCatalogAsync(IModuleData mod, SemVersionStyles style, string? archiveFile)
{
/*
* Builds the index.json file for the module. It
* contains an array of projects and their metadata
*/
string moduleVersion = mod.GetModuleCiVersion(config.DefaultCiVersion, style);
using MemoryStream ms = new();
using (Utf8JsonWriter writer = new(ms))
{
//Open initial object
writer.WriteStartObject();
InitModuleFile(writer, mod);
//Add the archive path if it was created in the module
if (archiveFile != null)
{
writer.WriteStartObject("archive");
//Archive path is in the build directory
writer.WriteString("path", config.SourceArchiveName);
//Get the checksum of the archive
string checksum = await new FileInfo(archiveFile).ComputeFileHashStringAsync();
writer.WriteString(config.HashFuncName, checksum);
writer.WriteString("sha_file", $"{config.SourceArchiveName}.{config.HashFuncName}");
//If signing is enabled, add the signature file (it is constant)
if (SignEnabled)
{
writer.WriteString("signature", $"{config.SourceArchiveName}.sig");
}
writer.WriteEndObject();
}
//Build project array
writer.WriteStartArray("projects");
foreach (IProject project in mod.Projects)
{
//start object for each project
writer.WriteStartObject();
//Write the project info
await WriteProjectInfoAsync(mod.FileManager, project, mod.Repository.Head.Tip.Sha, moduleVersion, writer);
writer.WriteEndObject();
}
writer.WriteEndArray();
//Close object
writer.WriteEndObject();
writer.Flush();
}
ms.Seek(0, SeekOrigin.Begin);
await mod.FileManager.WriteFileAsync(ModuleFileType.Catalog, ms.ToArray());
}
private static async Task BuildModuleVersionHistory(IModuleData mod)
{
/*
* Builds the index.json file for the module. It
* contains an array of projects and their metadata
*/
using MemoryStream ms = new();
using (Utf8JsonWriter writer = new(ms))
{
//Open initial object
writer.WriteStartObject();
InitModuleFile(writer, mod);
//Set the head pointer to the latest commit we build
writer.WriteString("head", mod.Repository.Head.Tip.Sha);
//Build project array
writer.WriteStartArray("versions");
//Write all git hashes from head back to the first commit
foreach (Commit commit in mod.Repository.Commits)
{
writer.WriteStringValue(commit.Sha);
}
writer.WriteEndArray();
//Releases will be an array of objects containing the tag and the hash
writer.WriteStartArray("releases");
//Write all git tags
foreach (Tag tag in mod.Repository.Tags.OrderByDescending(static p => p.FriendlyName))
{
writer.WriteStartObject();
writer.WriteString("tag", tag.FriendlyName);
writer.WriteString("hash", tag.Target.Sha);
writer.WriteEndObject();
}
writer.WriteEndArray();
//Close object
writer.WriteEndObject();
writer.Flush();
}
ms.Seek(0, SeekOrigin.Begin);
await mod.FileManager.WriteFileAsync(ModuleFileType.VersionHistory, ms.ToArray());
}
/*
* Builds the project's git history file and publishes it to the module file manager
*
* Also updates the latest hash file
*/
private static async Task BuildModuleGitHistoryAsync(IModuleData mod)
{
using MemoryStream ms = new();
using (Utf8JsonWriter writer = new(ms))
{
//Open initial object
writer.WriteStartObject();
InitModuleFile(writer, mod);
//Write the head commit
writer.WriteStartObject("head");
writer.WriteString("branch", mod.Repository.Head.FriendlyName);
WriteSingleCommit(writer, mod.Repository.Head.Tip);
writer.WriteEndObject();
//Write commit history
WriteCommitHistory(writer, mod.Repository);
//Close object
writer.WriteEndObject();
writer.Flush();
}
ms.Seek(0, SeekOrigin.Begin);
await mod.FileManager.WriteFileAsync(ModuleFileType.GitHistory, ms.ToArray());
await mod.FileManager.WriteFileAsync(
ModuleFileType.LatestHash,
Encoding.UTF8.GetBytes(mod.Repository.Head.Tip.Sha)
);
}
/*
* Captures all of the project artiacts and copies them to the module output directory
*/
private async Task CopyProjectOutputToModuleOutputAsync(IModuleData mod)
{
//Copy build artifacts to module output directory
await mod.Projects.RunAllAsync(project =>
{
//Get all output files from the project build, and copy them to the module output directory
return project.GetProjectBuildFiles(config)
.RunAllAsync(artifact => mod.FileManager.CopyArtifactToOutputAsync(project, artifact));
});
/*
* If signing is enabled, we can sign all project files synchronously
*/
if (SignEnabled)
{
Log.Information("GPG Siginig is enabled, signing all artifacts for module {mod}", mod.ModuleName);
/*
* Get all of the artifacts from the module's projects that match the target output
* file type, and sign them
*/
IEnumerable artifacts = mod.Projects.SelectMany(
p => mod.FileManager.GetArtifactOutputDir(p)
.EnumerateFiles(config.OutputFileType, SearchOption.TopDirectoryOnly)
);
//Sign synchronously
foreach (FileInfo artifact in artifacts)
{
await signer.SignFileAsync(artifact);
}
}
}
private static void InitModuleFile(Utf8JsonWriter writer, IModuleData mod)
{
//Set object name
writer.WriteString("module_name", mod.ModuleName);
//Modified date
writer.WriteString("modifed_date", DateTime.UtcNow);
}
private static void WriteCommitHistory(Utf8JsonWriter writer, Repository repo)
{
writer.WriteStartArray("commits");
//Write commit history for current repo
foreach (Commit commit in repo.Head.Commits)
{
writer.WriteStartObject();
WriteSingleCommit(writer, commit);
writer.WriteEndObject();
}
writer.WriteEndArray();
//Write tag history for current repo
writer.WriteStartArray("tags");
foreach (Tag tag in repo.Tags)
{
//clamp message length and ellipsis if too long
string? message = tag.Annotation?.Message;
if (message != null && message.Length > 120)
{
message = $"{message[..120]}...";
}
writer.WriteStartObject();
writer.WriteString("name", tag.FriendlyName);
writer.WriteString("sha", tag.Target.Sha);
writer.WriteString("message", message);
writer.WriteString("author", tag.Annotation?.Tagger.Name);
writer.WriteString("date", tag.Annotation?.Tagger.When ?? default);
writer.WriteEndObject();
}
writer.WriteEndArray();
}
private static void WriteSingleCommit(Utf8JsonWriter writer, Commit commit)
{
writer.WriteString("sha", commit.Sha);
writer.WriteString("message", commit.Message);
writer.WriteString("author", commit.Author.Name);
writer.WriteString("commiter", commit.Committer.Name);
writer.WriteString("date", commit.Committer.When);
writer.WriteString("message_short", commit.MessageShort);
}
///
/// Builds and writes the projects information to the
///
///
///
/// The to write the project
/// information to
///
/// A task that completes when the write operation has completed
private async Task WriteProjectInfoAsync(IModuleFileManager man, IProject project, string latestSha, string version, Utf8JsonWriter writer)
{
//Reload the project dom after execute because semversion may be updated after build step
if (project is ModuleProject mp)
{
await mp.LoadProjectDom();
}
writer.WriteString("name", project.ProjectName);
writer.WriteString("repo_url", project.ProjectData.RepoUrl);
writer.WriteString("description", project.ProjectData.Description);
writer.WriteString("version", version);
writer.WriteString("copyright", project.ProjectData.Copyright);
writer.WriteString("author", project.ProjectData.Authors);
writer.WriteString("product", project.ProjectData.Product);
writer.WriteString("company", project.ProjectData.CompanyName);
writer.WriteString("commit", latestSha);
//Write target framework if it exsits
writer.WriteString("target_framework", project.ProjectData["TargetFramework"]);
//Start file array
writer.WriteStartArray("files");
//Get only tar files, do not include the sha files
foreach (FileInfo output in GetProjOutputFiles(man, project))
{
//beging file object
writer.WriteStartObject();
writer.WriteString("name", output.Name);
writer.WriteString("path", $"{project.GetSafeProjectName()}/{output.Name}");
writer.WriteString("date", output.LastWriteTimeUtc);
writer.WriteNumber("size", output.Length);
//Compute the file hash
string hashHex = await output.ComputeFileHashStringAsync();
writer.WriteString(config.HashFuncName, hashHex);
//Path to sha-file
writer.WriteString("sha_file", $"{project.GetSafeProjectName()}/{output.Name}.{config.HashFuncName}");
writer.WriteEndObject();
}
writer.WriteEndArray();
}
private IEnumerable GetProjOutputFiles(IModuleFileManager man, IProject project)
{
return man.GetArtifactOutputDir(project)
.EnumerateFiles("*.*", SearchOption.TopDirectoryOnly)
.Where(p => p.Extension != $".{config.HashFuncName}");
}
///
/// Copies the source git archive tgz file to the output directory
///
///
///
///
private async Task CopySourceArchiveToOutput(IModuleData mod, IModuleFileManager man)
{
//Try to get a source archive in the module directory
string? archiveFile = Directory.EnumerateFiles(mod.Repository.Info.WorkingDirectory, config.SourceArchiveName, SearchOption.TopDirectoryOnly).FirstOrDefault();
//If archive is null ignore and continue
if (string.IsNullOrWhiteSpace(archiveFile))
{
Log.Information("No archive file found for module {mod}", mod.ModuleName);
return null;
}
Log.Information("Found source archive for module {mod}, copying to output...", mod.ModuleName);
//Otherwise copy to output
byte[] archive = await File.ReadAllBytesAsync(archiveFile);
FileInfo output = await man.WriteFileAsync(ModuleFileType.Archive, archive);
//Compute the hash of the file
await output.ComputeFileHashAsync(config.HashFuncName);
if (SignEnabled)
{
//Sign the file if signing is enabled
await signer.SignFileAsync(output);
}
return archiveFile;
}
}
}