diff options
Diffstat (limited to 'src/Publishing')
| -rw-r--r-- | src/Publishing/BuildPublisher.cs | 434 | ||||
| -rw-r--r-- | src/Publishing/FtpUploadManager.cs | 98 | ||||
| -rw-r--r-- | src/Publishing/GpgSigner.cs | 54 | ||||
| -rw-r--r-- | src/Publishing/MinioUploadManager.cs | 43 | ||||
| -rw-r--r-- | src/Publishing/SleetFeedManager.cs | 52 |
5 files changed, 681 insertions, 0 deletions
diff --git a/src/Publishing/BuildPublisher.cs b/src/Publishing/BuildPublisher.cs new file mode 100644 index 0000000..67807d5 --- /dev/null +++ b/src/Publishing/BuildPublisher.cs @@ -0,0 +1,434 @@ +using System; +using System.IO; +using System.Linq; +using System.Text; +using System.Text.Json; +using System.Threading.Tasks; +using System.Collections.Generic; + +using LibGit2Sharp; + +using Semver; + +using VNLib.Tools.Build.Executor.Constants; +using VNLib.Tools.Build.Executor.Model; +using VNLib.Tools.Build.Executor.Extensions; +using VNLib.Tools.Build.Executor.Projects; +using static VNLib.Tools.Build.Executor.Constants.Config; + +namespace VNLib.Tools.Build.Executor.Publishing +{ + + public sealed class BuildPublisher(BuildConfig config, GpgSigner signer) + { + public bool SignEnabled => signer.IsEnabled; + + /// <summary> + /// Prepares the module output and its collection of file details for publishing + /// then runs the upload step + /// </summary> + /// <param name="module"></param> + /// <returns>A task that completes when the module's output has been created</returns> + public async Task PrepareModuleOutput(IModuleData module) + { + //Copy project artifacts to output directory + await CopyProjectOutputToModuleOutputAsync(module); + + //Copy source archive + string? archiveFile = await CopySourceArchiveToOutput(module, module.FileManager); + + Log.Information("Building module {mod} catalog and git history", module.ModuleName); + + //Build module catalog + await BuildModuleCatalogAsync(module, config.SemverStyle, archiveFile); + + Log.Information("Building module {mod} git history", module.ModuleName); + + //Build git history + await BuildModuleGitHistoryAsync(module); + + Log.Information("Building module {mod} version history", module.ModuleName); + + //build version history + await BuildModuleVersionHistory(module); + + Log.Information("Moving module {mod} artifacts to the output", module.ModuleName); + } + + /// <summary> + /// Uploads the modules output to the remote + /// </summary> + /// <param name="module">The module containing the information to upload</param> + /// <returns></returns> + public Task UploadModuleOutput(IUploadManager Uploader, IModuleData module) + { + Log.Information("Uploading module {mod}", module.ModuleName); + + //Upload the entire output directory + return Uploader.UploadDirectoryAsync(module.FileManager.OutputDir); + } + + /* + * Builds the project catalog file and publishes it to the module file manager + */ + private async Task BuildModuleCatalogAsync(IModuleData mod, SemVersionStyles style, string? archiveFile) + { + /* + * Builds the index.json file for the module. It + * contains an array of projects and their metadata + */ + + string moduleVersion = mod.GetModuleCiVersion(config.DefaultCiVersion, style); + + using MemoryStream ms = new(); + + using (Utf8JsonWriter writer = new(ms)) + { + //Open initial object + writer.WriteStartObject(); + + InitModuleFile(writer, mod); + + //Add the archive path if it was created in the module + if (archiveFile != null) + { + writer.WriteStartObject("archive"); + + //Archive path is in the build directory + writer.WriteString("path", config.SourceArchiveName); + + //Get the checksum of the archive + string checksum = await new FileInfo(archiveFile).ComputeFileHashStringAsync(); + writer.WriteString(config.HashFuncName, checksum); + writer.WriteString("sha_file", $"{config.SourceArchiveName}.{config.HashFuncName}"); + + //If signing is enabled, add the signature file (it is constant) + if (SignEnabled) + { + writer.WriteString("signature", $"{config.SourceArchiveName}.sig"); + } + + writer.WriteEndObject(); + } + + //Build project array + writer.WriteStartArray("projects"); + + foreach (IProject project in mod.Projects) + { + //start object for each project + writer.WriteStartObject(); + + //Write the project info + await WriteProjectInfoAsync(mod.FileManager, project, mod.Repository.Head.Tip.Sha, moduleVersion, writer); + + writer.WriteEndObject(); + } + + writer.WriteEndArray(); + + //Close object + writer.WriteEndObject(); + + writer.Flush(); + } + + ms.Seek(0, SeekOrigin.Begin); + + await mod.FileManager.WriteFileAsync(ModuleFileType.Catalog, ms.ToArray()); + } + + private static async Task BuildModuleVersionHistory(IModuleData mod) + { + /* + * Builds the index.json file for the module. It + * contains an array of projects and their metadata + */ + + using MemoryStream ms = new(); + + using (Utf8JsonWriter writer = new(ms)) + { + //Open initial object + writer.WriteStartObject(); + + InitModuleFile(writer, mod); + + //Set the head pointer to the latest commit we build + writer.WriteString("head", mod.Repository.Head.Tip.Sha); + + //Build project array + writer.WriteStartArray("versions"); + + //Write all git hashes from head back to the first commit + foreach (Commit commit in mod.Repository.Commits) + { + writer.WriteStringValue(commit.Sha); + } + + writer.WriteEndArray(); + + //Releases will be an array of objects containing the tag and the hash + writer.WriteStartArray("releases"); + + //Write all git tags + foreach (Tag tag in mod.Repository.Tags.OrderByDescending(static p => p.FriendlyName)) + { + writer.WriteStartObject(); + + writer.WriteString("tag", tag.FriendlyName); + writer.WriteString("hash", tag.Target.Sha); + writer.WriteEndObject(); + } + + writer.WriteEndArray(); + + //Close object + writer.WriteEndObject(); + + writer.Flush(); + } + + ms.Seek(0, SeekOrigin.Begin); + + await mod.FileManager.WriteFileAsync(ModuleFileType.VersionHistory, ms.ToArray()); + } + + + /* + * Builds the project's git history file and publishes it to the module file manager + * + * Also updates the latest hash file + */ + private static async Task BuildModuleGitHistoryAsync(IModuleData mod) + { + using MemoryStream ms = new(); + + using (Utf8JsonWriter writer = new(ms)) + { + //Open initial object + writer.WriteStartObject(); + + InitModuleFile(writer, mod); + + //Write the head commit + writer.WriteStartObject("head"); + writer.WriteString("branch", mod.Repository.Head.FriendlyName); + WriteSingleCommit(writer, mod.Repository.Head.Tip); + writer.WriteEndObject(); + + //Write commit history + WriteCommitHistory(writer, mod.Repository); + + //Close object + writer.WriteEndObject(); + + writer.Flush(); + } + + ms.Seek(0, SeekOrigin.Begin); + + await mod.FileManager.WriteFileAsync(ModuleFileType.GitHistory, ms.ToArray()); + + await mod.FileManager.WriteFileAsync( + ModuleFileType.LatestHash, + Encoding.UTF8.GetBytes(mod.Repository.Head.Tip.Sha) + ); + } + + /* + * Captures all of the project artiacts and copies them to the module output directory + */ + private async Task CopyProjectOutputToModuleOutputAsync(IModuleData mod) + { + //Copy build artifacts to module output directory + await mod.Projects.RunAllAsync(project => + { + //Get all output files from the project build, and copy them to the module output directory + return project.GetProjectBuildFiles(config) + .RunAllAsync(artifact => mod.FileManager.CopyArtifactToOutputAsync(project, artifact)); + }); + + /* + * If signing is enabled, we can sign all project files synchronously + */ + if (SignEnabled) + { + Log.Information("GPG Siginig is enabled, signing all artifacts for module {mod}", mod.ModuleName); + + /* + * Get all of the artifacts from the module's projects that match the target output + * file type, and sign them + */ + IEnumerable<FileInfo> artifacts = mod.Projects.SelectMany( + p => mod.FileManager.GetArtifactOutputDir(p) + .EnumerateFiles(config.OutputFileType, SearchOption.TopDirectoryOnly) + ); + + //Sign synchronously + foreach (FileInfo artifact in artifacts) + { + await signer.SignFileAsync(artifact); + } + } + } + + + private static void InitModuleFile(Utf8JsonWriter writer, IModuleData mod) + { + //Set object name + writer.WriteString("module_name", mod.ModuleName); + //Modified date + writer.WriteString("modifed_date", DateTime.UtcNow); + } + + private static void WriteCommitHistory(Utf8JsonWriter writer, Repository repo) + { + writer.WriteStartArray("commits"); + + //Write commit history for current repo + foreach (Commit commit in repo.Head.Commits) + { + writer.WriteStartObject(); + + WriteSingleCommit(writer, commit); + + writer.WriteEndObject(); + } + + writer.WriteEndArray(); + + //Write tag history for current repo + writer.WriteStartArray("tags"); + + foreach (Tag tag in repo.Tags) + { + //clamp message length and ellipsis if too long + string? message = tag.Annotation?.Message; + if (message != null && message.Length > 120) + { + message = $"{message[..120]}..."; + } + + writer.WriteStartObject(); + writer.WriteString("name", tag.FriendlyName); + writer.WriteString("sha", tag.Target.Sha); + writer.WriteString("message", message); + writer.WriteString("author", tag.Annotation?.Tagger.Name); + writer.WriteString("date", tag.Annotation?.Tagger.When ?? default); + writer.WriteEndObject(); + } + + writer.WriteEndArray(); + } + + private static void WriteSingleCommit(Utf8JsonWriter writer, Commit commit) + { + writer.WriteString("sha", commit.Sha); + writer.WriteString("message", commit.Message); + writer.WriteString("author", commit.Author.Name); + writer.WriteString("commiter", commit.Committer.Name); + writer.WriteString("date", commit.Committer.When); + writer.WriteString("message_short", commit.MessageShort); + } + + /// <summary> + /// Builds and writes the projects information to the <see cref="Utf8JsonWriter"/> + /// </summary> + /// <param name="project"></param> + /// <param name="writer"> + /// The <see cref="Utf8JsonWriter"/> to write the project + /// information to + /// </param> + /// <returns>A task that completes when the write operation has completed</returns> + private async Task WriteProjectInfoAsync(IModuleFileManager man, IProject project, string latestSha, string version, Utf8JsonWriter writer) + { + //Reload the project dom after execute because semversion may be updated after build step + if (project is ModuleProject mp) + { + await mp.LoadProjectDom(); + } + + writer.WriteString("name", project.ProjectName); + writer.WriteString("repo_url", project.ProjectData.RepoUrl); + writer.WriteString("description", project.ProjectData.Description); + writer.WriteString("version", version); + writer.WriteString("copyright", project.ProjectData.Copyright); + writer.WriteString("author", project.ProjectData.Authors); + writer.WriteString("product", project.ProjectData.Product); + writer.WriteString("company", project.ProjectData.CompanyName); + writer.WriteString("commit", latestSha); + //Write target framework if it exsits + writer.WriteString("target_framework", project.ProjectData["TargetFramework"]); + + //Start file array + writer.WriteStartArray("files"); + + //Get only tar files, do not include the sha files + foreach (FileInfo output in GetProjOutputFiles(man, project)) + { + //beging file object + writer.WriteStartObject(); + + writer.WriteString("name", output.Name); + writer.WriteString("path", $"{project.GetSafeProjectName()}/{output.Name}"); + writer.WriteString("date", output.LastWriteTimeUtc); + writer.WriteNumber("size", output.Length); + + //Compute the file hash + string hashHex = await output.ComputeFileHashStringAsync(); + writer.WriteString(config.HashFuncName, hashHex); + + //Path to sha-file + writer.WriteString("sha_file", $"{project.GetSafeProjectName()}/{output.Name}.{config.HashFuncName}"); + + writer.WriteEndObject(); + } + + writer.WriteEndArray(); + } + + private IEnumerable<FileInfo> GetProjOutputFiles(IModuleFileManager man, IProject project) + { + return man.GetArtifactOutputDir(project) + .EnumerateFiles("*.*", SearchOption.TopDirectoryOnly) + .Where(p => p.Extension != $".{config.HashFuncName}"); + } + + /// <summary> + /// Copies the source git archive tgz file to the output directory + /// </summary> + /// <param name="mod"></param> + /// <param name="man"></param> + /// <returns></returns> + private async Task<string?> CopySourceArchiveToOutput(IModuleData mod, IModuleFileManager man) + { + //Try to get a source archive in the module directory + string? archiveFile = Directory.EnumerateFiles(mod.Repository.Info.WorkingDirectory, config.SourceArchiveName, SearchOption.TopDirectoryOnly).FirstOrDefault(); + + //If archive is null ignore and continue + if (string.IsNullOrWhiteSpace(archiveFile)) + { + Log.Information("No archive file found for module {mod}", mod.ModuleName); + return null; + } + + Log.Information("Found source archive for module {mod}, copying to output...", mod.ModuleName); + + //Otherwise copy to output + byte[] archive = await File.ReadAllBytesAsync(archiveFile); + FileInfo output = await man.WriteFileAsync(ModuleFileType.Archive, archive); + + //Compute the hash of the file + await output.ComputeFileHashAsync(config.HashFuncName); + + if (SignEnabled) + { + //Sign the file if signing is enabled + await signer.SignFileAsync(output); + } + + return archiveFile; + } + } +}
\ No newline at end of file diff --git a/src/Publishing/FtpUploadManager.cs b/src/Publishing/FtpUploadManager.cs new file mode 100644 index 0000000..a955bb8 --- /dev/null +++ b/src/Publishing/FtpUploadManager.cs @@ -0,0 +1,98 @@ +using System; +using System.IO; + +using FluentFTP; + +using System.Diagnostics.CodeAnalysis; +using System.Threading.Tasks; + +using VNLib.Tools.Build.Executor.Model; +using VNLib.Tools.Build.Executor.Constants; + +namespace VNLib.Tools.Build.Executor.Publishing +{ + internal sealed class FtpUploadManager(AsyncFtpClient client, string remotePath) : IUploadManager + { + public async Task UploadDirectoryAsync(string path) + { + path = Directory.GetParent(path)!.FullName; + + await client.AutoConnect(); + + var res = await client.UploadDirectory( + localFolder: path, + remoteFolder: remotePath, + FtpFolderSyncMode.Update, + FtpRemoteExists.Overwrite, + FtpVerify.Throw | FtpVerify.Retry + ); + + foreach(FtpResult fileResult in res) + { + switch (fileResult.ToStatus()) + { + case FtpStatus.Success: + Config.Log.Information( + "Uploaded {size} bytes, {0} -> {1}", + fileResult.Size, + fileResult.LocalPath, + fileResult.RemotePath + ); + break; + + case FtpStatus.Skipped: + Config.Log.Information("Skipped {0} -> {1}", fileResult.LocalPath, fileResult.RemotePath); + break; + + case FtpStatus.Failed: + Config.Log.Warning( + "Failed to upload {0}, reason: {exp}", + fileResult.LocalPath, + fileResult.Exception?.Message + ); + break; + } + } + } + + [return: NotNullIfNotNull(nameof(serverAddress))] + public static IUploadManager? Create(string? serverAddress) + { + if(string.IsNullOrWhiteSpace(serverAddress)) + { + return null; + } + + //Convert to uri, this may throw but this is currently the best way to validate the address + Uri serverUri = new(serverAddress); + + //Initlaize the client + AsyncFtpClient client = new() + { + Host = serverUri.Host, + Port = serverUri.Port, + + + Config = new() + { + LogToConsole = Config.Log.IsEnabled(Serilog.Events.LogEventLevel.Verbose), + + //Disable senstive logging in case running in automated CI pipelines where logs may be published + LogUserName = false, + LogPassword = false, + //EncryptionMode = FtpEncryptionMode.Auto, + RetryAttempts = 3, + }, + + Credentials = new() + { + //Pull credentials from the environment instead of command line + UserName = Environment.GetEnvironmentVariable("FTP_USERNAME"), + Password = Environment.GetEnvironmentVariable("FTP_PASSWORD") + }, + }; + + return new FtpUploadManager(client, serverUri.LocalPath); + } + } +}
\ No newline at end of file diff --git a/src/Publishing/GpgSigner.cs b/src/Publishing/GpgSigner.cs new file mode 100644 index 0000000..3ed216a --- /dev/null +++ b/src/Publishing/GpgSigner.cs @@ -0,0 +1,54 @@ +using System; +using System.IO; +using System.Threading.Tasks; +using System.Collections.Generic; + +using VNLib.Tools.Build.Executor.Constants; + +namespace VNLib.Tools.Build.Executor.Publishing +{ + public sealed class GpgSigner(bool enabled, string? defaultKey) + { + public bool IsEnabled { get; } = enabled; + + public async Task SignFileAsync(FileInfo file) + { + if (!IsEnabled) + { + return; + } + + List<string> args = [ + "--detach-sign" + ]; + + if (!string.IsNullOrWhiteSpace(defaultKey)) + { + //Set the preferred key + args.Add("--default-key"); + args.Add(defaultKey); + } + + //Add input file + args.Add(file.FullName); + + //Delete an original file + string sigFile = $"{file.FullName}.sig"; + if (File.Exists(sigFile)) + { + File.Delete(sigFile); + } + + int result = await Utils.RunProcessAsync("gpg", null, args.ToArray()); + + switch (result) + { + case 2: + case 0: + break; + default: + throw new BuildFailedException($"Failed to sign file {file.FullName}"); + } + } + } +}
\ No newline at end of file diff --git a/src/Publishing/MinioUploadManager.cs b/src/Publishing/MinioUploadManager.cs new file mode 100644 index 0000000..502392f --- /dev/null +++ b/src/Publishing/MinioUploadManager.cs @@ -0,0 +1,43 @@ +using System.Threading.Tasks; +using System.Diagnostics.CodeAnalysis; + +using VNLib.Tools.Build.Executor.Model; + +using static VNLib.Tools.Build.Executor.Constants.Utils; + +namespace VNLib.Tools.Build.Executor.Publishing +{ + + internal sealed class MinioUploadManager : IUploadManager + { + private readonly string _minioPath; + + private MinioUploadManager(string minioPath) => _minioPath = minioPath; + + public async Task UploadDirectoryAsync(string path) + { + //Recursivley copy all files in the working directory + string[] args = + { + "cp", + "--recursive", + ".", + _minioPath + }; + + //Set working dir to the supplied dir path, and run the command + int result = await RunProcessAsync("mc", path, args); + + if (result != 0) + { + throw new BuildFailedException($"Failed to upload directory {path} with status code {result:x}"); + } + } + + [return: NotNullIfNotNull(nameof(uploadPath))] + public static IUploadManager? Create(string? uploadPath) + { + return string.IsNullOrWhiteSpace(uploadPath) ? null : new MinioUploadManager(uploadPath); + } + } +}
\ No newline at end of file diff --git a/src/Publishing/SleetFeedManager.cs b/src/Publishing/SleetFeedManager.cs new file mode 100644 index 0000000..997dcf2 --- /dev/null +++ b/src/Publishing/SleetFeedManager.cs @@ -0,0 +1,52 @@ +using System; +using System.Diagnostics.CodeAnalysis; +using System.IO; +using System.Text.Json; +using VNLib.Tools.Build.Executor.Model; + +namespace VNLib.Tools.Build.Executor.Publishing +{ + internal sealed class SleetFeedManager : IFeedManager + { + private readonly string SleetConfigFile; + + ///<inheritdoc/> + public string FeedOutputDir { get; } + + private SleetFeedManager(string indexFilex, string outputDir) + { + //Search for the sleet file in the build dir + SleetConfigFile = indexFilex; + FeedOutputDir = outputDir; + } + + ///<inheritdoc/> + public void AddVariables(TaskfileVars vars) + { + vars.Set("SLEET_DIR", FeedOutputDir); + vars.Set("SLEET_CONFIG_PATH", SleetConfigFile); + } + + /// <summary> + /// Attempts to create a new sleet feed manager from the given directory index + /// if the index contains a sleet feed. Returns null if no sleet feed was found + /// </summary> + /// <returns>The feed manager if found, null otherwise</returns> + /// <exception cref="FileNotFoundException"></exception> + /// <exception cref="ArgumentException"></exception> + [return: NotNullIfNotNull(nameof(feedPath))] + public static IFeedManager? GetSleetFeed(string? feedPath) + { + if (string.IsNullOrWhiteSpace(feedPath)) + { + return null; + } + + //Read the sleet config file + byte[] sleetIndexFile = File.ReadAllBytes(feedPath); + using JsonDocument doc = JsonDocument.Parse(sleetIndexFile); + string rootDir = doc.RootElement.GetProperty("root").GetString() ?? throw new ArgumentException("The sleet output directory was not specified"); + return new SleetFeedManager(feedPath, rootDir); + } + } +}
\ No newline at end of file |