diff options
56 files changed, 4032 insertions, 1 deletions
@@ -360,4 +360,6 @@ MigrationBackup/ .ionide/ # Fody - auto-generated XML schema -FodyWeavers.xsd
\ No newline at end of file +FodyWeavers.xsd + +*.licenseheader
\ No newline at end of file diff --git a/.onedev-buildspec.yml b/.onedev-buildspec.yml new file mode 100644 index 0000000..db67422 --- /dev/null +++ b/.onedev-buildspec.yml @@ -0,0 +1,39 @@ +version: 33 +jobs: +- name: Push to GitHub + steps: + - !PushRepository + name: GitHub Sync + remoteUrl: https://github.com/VnUgE/vnbuild.git + userName: VnUgE + passwordSecret: git-access-token + force: false + condition: ALL_PREVIOUS_STEPS_WERE_SUCCESSFUL + triggers: + - !TagCreateTrigger + projects: vnbuild + - !BranchUpdateTrigger + projects: vnbuild + retryCondition: never + maxRetries: 3 + retryDelay: 30 + timeout: 3600 +- name: Pull from GitHub + steps: + - !PullRepository + name: Sync from GitHub + remoteUrl: https://github.com/VnUgE/vnbuild.git + userName: VnUgE + passwordSecret: git-access-token + refs: refs/heads/* refs/tags/* + withLfs: false + force: false + condition: ALL_PREVIOUS_STEPS_WERE_SUCCESSFUL + triggers: + - !ScheduleTrigger + cronExpression: 0 15 10 ? * * + projects: vnbuild + retryCondition: never + maxRetries: 3 + retryDelay: 30 + timeout: 3600 diff --git a/GitVersion.yml b/GitVersion.yml new file mode 100644 index 0000000..ebdfba1 --- /dev/null +++ b/GitVersion.yml @@ -0,0 +1,6 @@ +assembly-versioning-scheme: MajorMinorPatch +mode: ContinuousDeployment +branches: {} +ignore: + sha: [] +merge-message-formats: {} @@ -0,0 +1,293 @@ +Copyright (c) 2024 Vaughn Nugent + +Contact information + Name: Vaughn Nugent + Email: vnpublic[at]proton.me + Website: https://www.vaughnnugent.com + +The software in this repository is licensed under the GNU GPL version 2.0 (or any later version). + +SPDX-License-Identifier: GPL-2.0-or-later + +License-Text: + +GNU GENERAL PUBLIC LICENSE + Version 2, June 1991 + + Copyright (C) 1989, 1991 Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The licenses for most software are designed to take away your +freedom to share and change it. By contrast, the GNU General Public +License is intended to guarantee your freedom to share and change free +software--to make sure the software is free for all its users. This +General Public License applies to most of the Free Software +Foundation's software and to any other program whose authors commit to +using it. (Some other Free Software Foundation software is covered by +the GNU Lesser General Public License instead.) You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +this service if you wish), that you receive source code or can get it +if you want it, that you can change the software or use pieces of it +in new free programs; and that you know you can do these things. + + To protect your rights, we need to make restrictions that forbid +anyone to deny you these rights or to ask you to surrender the rights. +These restrictions translate to certain responsibilities for you if you +distribute copies of the software, or if you modify it. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must give the recipients all the rights that +you have. You must make sure that they, too, receive or can get the +source code. And you must show them these terms so they know their +rights. + + We protect your rights with two steps: (1) copyright the software, and +(2) offer you this license which gives you legal permission to copy, +distribute and/or modify the software. + + Also, for each author's protection and ours, we want to make certain +that everyone understands that there is no warranty for this free +software. If the software is modified by someone else and passed on, we +want its recipients to know that what they have is not the original, so +that any problems introduced by others will not reflect on the original +authors' reputations. + + Finally, any free program is threatened constantly by software +patents. We wish to avoid the danger that redistributors of a free +program will individually obtain patent licenses, in effect making the +program proprietary. To prevent this, we have made it clear that any +patent must be licensed for everyone's free use or not licensed at all. + + The precise terms and conditions for copying, distribution and +modification follow. + + GNU GENERAL PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. This License applies to any program or other work which contains +a notice placed by the copyright holder saying it may be distributed +under the terms of this General Public License. The "Program", below, +refers to any such program or work, and a "work based on the Program" +means either the Program or any derivative work under copyright law: +that is to say, a work containing the Program or a portion of it, +either verbatim or with modifications and/or translated into another +language. (Hereinafter, translation is included without limitation in +the term "modification".) Each licensee is addressed as "you". + +Activities other than copying, distribution and modification are not +covered by this License; they are outside its scope. The act of +running the Program is not restricted, and the output from the Program +is covered only if its contents constitute a work based on the +Program (independent of having been made by running the Program). +Whether that is true depends on what the Program does. + + 1. You may copy and distribute verbatim copies of the Program's +source code as you receive it, in any medium, provided that you +conspicuously and appropriately publish on each copy an appropriate +copyright notice and disclaimer of warranty; keep intact all the +notices that refer to this License and to the absence of any warranty; +and give any other recipients of the Program a copy of this License +along with the Program. + +You may charge a fee for the physical act of transferring a copy, and +you may at your option offer warranty protection in exchange for a fee. + + 2. You may modify your copy or copies of the Program or any portion +of it, thus forming a work based on the Program, and copy and +distribute such modifications or work under the terms of Section 1 +above, provided that you also meet all of these conditions: + + a) You must cause the modified files to carry prominent notices + stating that you changed the files and the date of any change. + + b) You must cause any work that you distribute or publish, that in + whole or in part contains or is derived from the Program or any + part thereof, to be licensed as a whole at no charge to all third + parties under the terms of this License. + + c) If the modified program normally reads commands interactively + when run, you must cause it, when started running for such + interactive use in the most ordinary way, to print or display an + announcement including an appropriate copyright notice and a + notice that there is no warranty (or else, saying that you provide + a warranty) and that users may redistribute the program under + these conditions, and telling the user how to view a copy of this + License. (Exception: if the Program itself is interactive but + does not normally print such an announcement, your work based on + the Program is not required to print an announcement.) + +These requirements apply to the modified work as a whole. If +identifiable sections of that work are not derived from the Program, +and can be reasonably considered independent and separate works in +themselves, then this License, and its terms, do not apply to those +sections when you distribute them as separate works. But when you +distribute the same sections as part of a whole which is a work based +on the Program, the distribution of the whole must be on the terms of +this License, whose permissions for other licensees extend to the +entire whole, and thus to each and every part regardless of who wrote it. + +Thus, it is not the intent of this section to claim rights or contest +your rights to work written entirely by you; rather, the intent is to +exercise the right to control the distribution of derivative or +collective works based on the Program. + +In addition, mere aggregation of another work not based on the Program +with the Program (or with a work based on the Program) on a volume of +a storage or distribution medium does not bring the other work under +the scope of this License. + + 3. You may copy and distribute the Program (or a work based on it, +under Section 2) in object code or executable form under the terms of +Sections 1 and 2 above provided that you also do one of the following: + + a) Accompany it with the complete corresponding machine-readable + source code, which must be distributed under the terms of Sections + 1 and 2 above on a medium customarily used for software interchange; or, + + b) Accompany it with a written offer, valid for at least three + years, to give any third party, for a charge no more than your + cost of physically performing source distribution, a complete + machine-readable copy of the corresponding source code, to be + distributed under the terms of Sections 1 and 2 above on a medium + customarily used for software interchange; or, + + c) Accompany it with the information you received as to the offer + to distribute corresponding source code. (This alternative is + allowed only for noncommercial distribution and only if you + received the program in object code or executable form with such + an offer, in accord with Subsection b above.) + +The source code for a work means the preferred form of the work for +making modifications to it. For an executable work, complete source +code means all the source code for all modules it contains, plus any +associated interface definition files, plus the scripts used to +control compilation and installation of the executable. However, as a +special exception, the source code distributed need not include +anything that is normally distributed (in either source or binary +form) with the major components (compiler, kernel, and so on) of the +operating system on which the executable runs, unless that component +itself accompanies the executable. + +If distribution of executable or object code is made by offering +access to copy from a designated place, then offering equivalent +access to copy the source code from the same place counts as +distribution of the source code, even though third parties are not +compelled to copy the source along with the object code. + + 4. You may not copy, modify, sublicense, or distribute the Program +except as expressly provided under this License. Any attempt +otherwise to copy, modify, sublicense or distribute the Program is +void, and will automatically terminate your rights under this License. +However, parties who have received copies, or rights, from you under +this License will not have their licenses terminated so long as such +parties remain in full compliance. + + 5. You are not required to accept this License, since you have not +signed it. However, nothing else grants you permission to modify or +distribute the Program or its derivative works. These actions are +prohibited by law if you do not accept this License. Therefore, by +modifying or distributing the Program (or any work based on the +Program), you indicate your acceptance of this License to do so, and +all its terms and conditions for copying, distributing or modifying +the Program or works based on it. + + 6. Each time you redistribute the Program (or any work based on the +Program), the recipient automatically receives a license from the +original licensor to copy, distribute or modify the Program subject to +these terms and conditions. You may not impose any further +restrictions on the recipients' exercise of the rights granted herein. +You are not responsible for enforcing compliance by third parties to +this License. + + 7. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), +conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot +distribute so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you +may not distribute the Program at all. For example, if a patent +license would not permit royalty-free redistribution of the Program by +all those who receive copies directly or indirectly through you, then +the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Program. + +If any portion of this section is held invalid or unenforceable under +any particular circumstance, the balance of the section is intended to +apply and the section as a whole is intended to apply in other +circumstances. + +It is not the purpose of this section to induce you to infringe any +patents or other property right claims or to contest validity of any +such claims; this section has the sole purpose of protecting the +integrity of the free software distribution system, which is +implemented by public license practices. Many people have made +generous contributions to the wide range of software distributed +through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing +to distribute software through any other system and a licensee cannot +impose that choice. + +This section is intended to make thoroughly clear what is believed to +be a consequence of the rest of this License. + + 8. If the distribution and/or use of the Program is restricted in +certain countries either by patents or by copyrighted interfaces, the +original copyright holder who places the Program under this License +may add an explicit geographical distribution limitation excluding +those countries, so that distribution is permitted only in or among +countries not thus excluded. In such case, this License incorporates +the limitation as if written in the body of this License. + + 9. The Free Software Foundation may publish revised and/or new versions +of the General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + +Each version is given a distinguishing version number. If the Program +specifies a version number of this License which applies to it and "any +later version", you have the option of following the terms and conditions +either of that version or of any later version published by the Free +Software Foundation. If the Program does not specify a version number of +this License, you may choose any version ever published by the Free Software +Foundation. + + 10. If you wish to incorporate parts of the Program into other free +programs whose distribution conditions are different, write to the author +to ask for permission. For software which is copyrighted by the Free +Software Foundation, write to the Free Software Foundation; we sometimes +make exceptions for this. Our decision will be guided by the two goals +of preserving the free status of all derivatives of our free software and +of promoting the sharing and reuse of software generally. + + NO WARRANTY + + 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY +FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN +OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES +PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED +OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS +TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE +PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, +REPAIR OR CORRECTION. + + 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR +REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, +INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING +OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED +TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY +YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER +PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE +POSSIBILITY OF SUCH DAMAGES. + + END OF TERMS AND CONDITIONS
\ No newline at end of file diff --git a/Module.Taskfile.yaml b/Module.Taskfile.yaml new file mode 100644 index 0000000..4afcbfa --- /dev/null +++ b/Module.Taskfile.yaml @@ -0,0 +1,45 @@ +# https://taskfile.dev + +version: '3' + +vars: + INT_DIR: '{{.SCRATCH_DIR}}/obj/{{.MODULE_NAME}}/' + MS_ARGS: '--sc false /p:RunAnalyzersDuringBuild=false /p:IntermediateOutputPath="{{.INT_DIR}}" /p:UseCommonOutputDirectory=true /p:BuildInParallel=true /p:MultiProcessorCompilation=true /p:ErrorOnDuplicatePublishOutputFiles=false' + +tasks: + #called by build pipeline to sync repo + update: + dir: '{{.USER_WORKING_DIR}}' + cmds: + - git reset --hard #clean up any local changes + - git remote update + - git pull origin {{.BRANCH_NAME}} --verify-signatures + #re-write semver after hard reset + - dotnet-gitversion.exe /updateprojectfiles + +#called by build pipeline to build module + build: + dir: '{{.USER_WORKING_DIR}}' + cmds: + + - for: [ win-x64, linux-x64, osx-x64, linux-arm64, linux-arm ] + cmd: powershell -Command 'dotnet publish -c debug -r {{ .ITEM }} {{.BUILD_FLAGS}} {{.MS_ARGS}}' + + #build release mode after all debug builds + - for: [ win-x64, linux-x64, osx-x64, linux-arm64, linux-arm ] + cmd: powershell -Command 'dotnet publish -c release -r {{ .ITEM }} {{.BUILD_FLAGS}} {{.MS_ARGS}}' + + postbuild_success: + cmds: + #git archive in the module directory + - git archive --format {{.ARCHIVE_FILE_FORMAT}} --output {{.ARCHIVE_FILE_NAME}} HEAD + + +#called by build pipeline to clean module + clean: + dir: '{{.USER_WORKING_DIR}}' + cmds: + #clean solution + - dotnet clean /p:BuildInParallel=true /p:MultiProcessorCompilation=true + - cmd: powershell -Command "rm {{ .ARCHIVE_FILE_NAME }} --Force" + ignore_error: true diff --git a/README.md b/README.md new file mode 100644 index 0000000..f9ec819 --- /dev/null +++ b/README.md @@ -0,0 +1,108 @@ +# vnbuild +*Automatically builds and deploys binaries from git-based configurable pipelines with proper indexing for web based deployments* + +## Introduction +I built this tool for repeatable builds, defined by the "code" in a repo via a command line interface, that includes integration with MSBuild solutions and projects, along with leaf projects defined by a `package.json` file. It needed to work well with multiple projects per repo (aka module). Next, it needed to publish the packages produced by the build step where they could be easily shared via a website. Finally I didn't want to be forced to use a huge CI system or someone else's servers to build my code. VNBuild was born in the winter of 2022 and has had incremental updates since. + +> [!WARNING] +> This tool relies on Typin, which has not been actively developed in multiple years. + +## Install +Follow the links below for software downloads and extended documentation. Releases are gzip tar archives that have sh256 sums and pgp signatures, along with complete source code and repository archive. + +**[Builds and Source](https://www.vaughnnugent.com/resources/software/modules/vnbuild)** Download the latest package for your operating system and architecture. +**[Docs and Articles](https://www.vaughnnugent.com/resources/software/articles?tags=docs,_vnbuild)** Read the documentation and articles to get started. + +(Fun fact: This project publishes itself!) + +## Basic Commands +- **update**: Updates the module and all projects within the module. +- **build**: Builds the module and all projects within the module. +- **publish**: Publishes the module and all projects within the module. +- **clean**: Cleans up the module and all projects within the module. +- **test**: Runs tests on the module and all projects within the module. + +**Always use the --help flag to get more information about a command** it will be more detailed than the information provided here. + +## Terminology +- **Module**: A self-contained git repository that has all of the necessary files to build a project or multiple projects as a single unit. +- **Project**: A single build target within a module. A module can have multiple projects. + +### Repo update + +### Building + +### Publishing + +#### GPG Signing +If the `--sign` flag is set during a publish command, the files produced by a single project will individually be signed, and the signature files located in the same output directory. + +### Cleaning + +## Taskfile.dev +vnbuild uses [Taskfile.dev](https://taskfile.dev) (installed on your machine) to actually execute the build steps within a module. + +### Taskfiles +In the top level of your module, you must include a file named `Module.Taskfile.yaml`. This taskfile will be responsible for running tasks at a module level. It has the same functions any project-level taskfile does, but gets called first, and it's error codes will be observed. This file will also be responsible for updating the module's repository via a named task `update`. + +You may **optionally** have one or more `Taskfile.yaml` file(s) that will be called at a project level for every discovered project within the module. This file will be responsible for running tasks at a project level. + +#### How Task is used +For example, when you run `vnbuild build`: +1. vnbuild will look for a `Module.Taskfile.yaml` file in the root of the module. +2. Task will be executed (with -t) to run the Module.Taskfile.yaml file's build command within the module's root directory. This command is rquired, and its return code will be observed. +3. vnbuild will then execute Task process in the directory of each project found in the module. +4. Task searches up the directory tree for a `Taskfile.yaml` file (similar to git) and executes the task named 'build' if a taskfile is found. The results of this command are observed. + +The same process is followed for `vnbuild publish` and `vnbuild clean` commands. + +Most projects (of the same programming language) within a "monorepo" have similar build/publish steps, so I often have a single Taskfile.yaml in the root of the module with "generic" steps, if any project needs to be treated differently, I will add a modified Taskfile.yaml to that project's directory. In the case of C# modules, building with solution files can be mutch faster than building each project manually, so in that case, your Module.Taskfile.yaml should handle that, same with a large CMake project as well. + +### Named tasks +vnbuild will execute named tasks within the Taskfile. The following tasks are required: + +- **update**: Task the runs a repository sync operation. Only called during an `update` command. (Only available within Module.Taskfile.yaml) +- **build**: The task that actually builds the project. Only called during a `build` command. +- **postbuild_success**: Called after a successful build task finished. Only called during a `build` command. +- **postbuild_failure**: Called after a failed build task finished. Only called during a `build` command. +- **test**: A task that runs tests on the project. Only called during a `test` command. (all exit codes are observed, nonzero exit codes are considered a failure) +- **publish**: A task the runs publish operations such as copying files to a deployment directory or adding to a directory. Only called during a `publish` command. +- **clean**: A task that cleans up any temporary files or directories created during the build process. Only called during a `clean` command. + +### Variables + +#### Global +BUILD_DIR - the build wide .build output directory +SCRATCH_DIR - the process wide shared scratch directory +UNIX_MS - the unix (ms) timestamp at the start of the build process +DATE - full normalized date and time string +HEAD_SHA - the sha1 of the current head pointer +BRANCH_NAME - the name of the branch currently pointed to by HEAD + +#### Module +MODULE_NAME - the name of the module solution +OUTPUT_DIR - the module's output directory +MODULE_DIR - the root dir of the module +SOLUTION_FILE_NAME - The name of the solution file (including the extension) +ARCHIVE_FILE_NAME - The git archive target file name +FULL_ARCHIVE_FILE_NAME - The full file path to the desired git archive +ARCHIVE_FILE_FORMAT - git archive format type +BUILD_VERSION - Calculated module semver + +#### Project +PROJECT_NAME - the name of the project +PROJECT_DIR - the root dir of the project +IS_PROJECT - 'True' or undefined if the call is from a project scope +SAFE_PROJ_NAME - The filesystem safe project name (removes any illegal filesystem characters and replaces them with hyphens) + +#### Available from .x_proj/package.json files +PROJ_VERSION - the version string +PROJ_DESCRIPTION - the description string +PROJ_AUTHOR - the author string +PROJ_COPYRIGHT - the copyright text from the project file +PROJ_COMPANY - The company name +RPOJ_URL - the project repository url +BINARY_DIR - relative directory of the binary output (set by package) + +## License +The software in this repository is licensed under the GNU GPL version 2.0 (or any later version). See the LICENSE files for more information.
\ No newline at end of file diff --git a/Taskfile.yaml b/Taskfile.yaml new file mode 100644 index 0000000..0e42d83 --- /dev/null +++ b/Taskfile.yaml @@ -0,0 +1,51 @@ +# https://taskfile.dev + +version: '3' + +vars: + INT_DIR: '{{.SCRATCH_DIR}}/obj/{{.MODULE_NAME}}/' + TARGET: '{{.USER_WORKING_DIR}}/bin' + RELEASE_DIR: "./bin/release/{{.TARGET_FRAMEWORK}}" + +tasks: + + #when build succeeds, archive the output into a tgz + postbuild_success: + dir: '{{.USER_WORKING_DIR}}' + cmds: + #pack up source code and put in output + - powershell -Command "Get-ChildItem -Include *.cs,*.csproj -Recurse | Where { \$_.FullName -notlike '*\obj\*' } | Resolve-Path -Relative | tar --files-from - -czf '{{.TARGET}}/src.tgz'" + + #run post in debug mode + - for: [ win-x64, linux-x64, osx-x64, linux-arm64, linux-arm ] + task: postbuild + vars: { BUILD_MODE: debug, TARGET_OS: '{{ .ITEM }}'} + + #remove uncessary files from the release dir + - powershell -Command "Get-ChildItem -Recurse '{{.RELEASE_DIR}}/' -Include *.pdb,*.xml | Remove-Item" + + - for: [ win-x64, linux-x64, osx-x64, linux-arm64, linux-arm ] + task: postbuild + vars: { BUILD_MODE: release, TARGET_OS: '{{ .ITEM }}'} + + postbuild: + internal: true + dir: '{{.USER_WORKING_DIR}}' + vars: + BUILD_DIR: "{{.USER_WORKING_DIR}}/bin/{{.BUILD_MODE}}/{{.TARGET_FRAMEWORK}}/{{.TARGET_OS}}/publish" + cmds: + #copy and readme to target + - cmd: cd .. && powershell -Command "Copy-Item -Path ./build.readme.txt -Destination '{{.BUILD_DIR}}/readme.txt'" + ignore_error: true + + #tar outputs + - cd "{{.BUILD_DIR}}" && tar -czf "{{.TARGET}}/{{.TARGET_OS}}-{{.BUILD_MODE}}.tgz" . + + #Remove the output dirs on clean + clean: + dir: '{{.USER_WORKING_DIR}}' + ignore_error: true + cmds: + - for: [ bin/, obj/ ] + cmd: powershell Remove-Item -Recurse './{{.TARGET}}'] +
\ No newline at end of file diff --git a/build.readme.txt b/build.readme.txt new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/build.readme.txt diff --git a/src/BuildPipeline.cs b/src/BuildPipeline.cs new file mode 100644 index 0000000..2435566 --- /dev/null +++ b/src/BuildPipeline.cs @@ -0,0 +1,305 @@ +using System; +using System.IO; +using System.Linq; +using System.Threading.Tasks; +using System.Collections.Generic; + +using Serilog; +using Serilog.Core; + +using VNLib.Tools.Build.Executor.Model; +using VNLib.Tools.Build.Executor.Modules; +using VNLib.Tools.Build.Executor.Extensions; +using VNLib.Tools.Build.Executor.Constants; + +namespace VNLib.Tools.Build.Executor +{ + + public sealed class BuildPipeline(Logger Log) : IDisposable + { + private readonly List<ModuleBase> _allModules = new(); + private readonly List<ModuleBase> _selected = new(); + private readonly LinkedList<ModuleBase> _outdatedModules = new(); + private readonly LinkedList<IProject> _modifiedProjects = new(); + private readonly TaskfileVars _taskVars = new(); + + /// <summary> + /// Loads a modules within the working directory + /// </summary> + /// <returns>A task that completes when all modules and child projects are loaded</returns> + public async Task LoadAsync(BuildConfig config, string[] only, string[] exclude, IFeedManager[] feeds) + { + //Init task variables + SetTaskVariables(config.Index, feeds); + + //Capture all modules within pwd + Log.Information("Discovering modules in {pwd}", config.Index.BaseDir.FullName); + + //Search for .git repos + DirectoryInfo[] moduleDirs = config.Index.BaseDir.EnumerateDirectories(".git", SearchOption.AllDirectories) + .Select(static s => s.Parent!) + .ToArray(); + + //Add modules + foreach(DirectoryInfo dir in moduleDirs) + { + _allModules.Add(new GitCodeModule(config, dir)); + } + + Log.Information("Found {c} modules, loading modules...", moduleDirs.Length); + + //Load all modules async and give them each a copy of our local task variables + await _allModules.RunAllAsync(p => p.LoadAsync(_taskVars.Clone())); + + //Only include desired modules + if (only.Length > 0) + { + Log.Information("Only including modules {mods}", only); + + ModuleBase[] onlyMods = _allModules.Where(m => only.Contains(m.ModuleName, StringComparer.OrdinalIgnoreCase)).ToArray(); + _selected.AddRange(onlyMods); + } + //Exclude given modules + else if(exclude.Length > 0) + { + Log.Information("Excluding modules {mods}", exclude); + + ModuleBase[] excludeMods = _allModules.Where(m => exclude.Contains(m.ModuleName, StringComparer.OrdinalIgnoreCase)).ToArray(); + _selected.AddRange(_allModules.Except(excludeMods)); + } + else + { + //Just all all modules to the list + _selected.AddRange(_allModules); + } + + Log.Information("The following modules will be processed\n{mods}", _selected.Select(m => m.ModuleName)); + } + + private void SetTaskVariables(IDirectoryIndex dirIndex, IFeedManager[] feeds) + { + //Configure variables + _taskVars.Set("BUILD_DIR", dirIndex.BuildDir.FullName); + _taskVars.Set("SCRATCH_DIR", dirIndex.ScratchDir.FullName); + _taskVars.Set("UNIX_MS", DateTimeOffset.UtcNow.ToUnixTimeMilliseconds().ToString()); + _taskVars.Set("DATE", DateTimeOffset.Now.ToString("d")); + + //Add all feed manager to task variables + Array.ForEach(feeds, f => f.AddVariables(_taskVars)); + } + + /// <summary> + /// Synchronizes all modules with their respective remote repositories + /// </summary> + /// <returns></returns> + public async Task DoStepUpdateSource() + { + //Clear outdated list before syncing sources + _outdatedModules.Clear(); + _modifiedProjects.Clear(); + + //Must sync source serially to prevent git errors + foreach (ModuleBase module in _selected) + { + //Sync source + await module.DoStepSyncSource(); + } + } + + /// <summary> + /// Prepares the build pipeline for building, finds for changes and determines dependencies + /// then prepares modules for building + /// </summary> + /// <returns></returns> + public async Task<bool> CheckForChangesAsync() + { + //Clear outdated list before syncing sources + _outdatedModules.Clear(); + _modifiedProjects.Clear(); + + //Conccurrently search for changes in all modules + await _selected.RunAllAsync(async m => + { + if (await m.CheckForChangesAsync()) + { + _outdatedModules.AddLast(m); + Log.Information("Module {m} MODIFIED. Queued for rebuild", m.ModuleName); + } + }); + + //if one or more modules have been modified, we need to determine dependencies + if (_outdatedModules.Count > 0) + { + //Get the initial list of projects that will be rebuilt + string[] outDatedProjects = _outdatedModules.SelectMany(static m => m.Projects.Where(static p => !p.UpToDate).Select(static p => p.ProjectFile.Name)).ToArray(); + + do + { + + /* + * Select only up-to-date modules + * that have external project references to outdated + * projects + */ + ModuleBase[] dependants = _selected.Where(m => !_outdatedModules.Contains(m)) + .Where( + m => m.GetExternalDependencies() + .Where(externProj => outDatedProjects.Contains(externProj)) + .Any()) + .ToArray(); + + //If there are no more dependants, exit loop + if (dependants.Length == 0) + { + break; + } + + //Add modules to oudated list + for (int i = 0; i < dependants.Length; i++) + { + Log.Information("Module {mod} OUTDATED because it depends on out-of-date modules", dependants[i].ModuleName); + _outdatedModules.AddLast(dependants[i]); + } + + //update outdated projects list to include projects from the newly outdated modules + outDatedProjects = dependants.SelectMany(static p => p.GetExternalDependencies()).ToArray(); + } + while (true); + } + + Log.Information("{c} modules detected source code changes", _outdatedModules.Count); + return _outdatedModules.Count > 0; + } + + public async Task DoStepBuild(bool force) + { + //Rebuild all modules + if (force) + { + //rebuild all selected modules + foreach (ModuleBase mod in _selected) + { + //Run each module independently + await BuildSingleModule(mod, Log); + } + } + else + { + if (_outdatedModules.Count == 0) + { + Log.Information("No modules detected changes"); + } + + //Only rebuild modified modules + foreach (ModuleBase mod in _outdatedModules) + { + //Run each module independently + await BuildSingleModule(mod, Log); + } + } + } + + static async Task BuildSingleModule(IBuildable module, ILogger log) + { + log.Information("Building module {m}", (module as ModuleBase)!.ModuleName); + + try + { + //Build module + await module.DoStepBuild(); + } + catch + { + //failure + await module.DoStepPostBuild(false); + throw; + } + + //Completed successfully, await the result of post-build + await module.DoStepPostBuild(true); + } + + public async Task OnPublishingAsync() + { + /* + * Exec publish step on modules in order incase they + * need to access synchronous resources + */ + + foreach(ModuleBase module in _selected) + { + await module.DoStepPublish(); + } + } + + public async Task PrepareOutputAsync(BuildPublisher publisher) + { + Log.Information("Preparing pipline output"); + + if(publisher.SignEnabled) + { + //Sign all modules synchronously so gpg-agent doesn't get overloaded + foreach (IModuleData module in _selected) + { + //Sign module + await publisher.PrepareModuleOutput(module); + } + } + else + { + await _selected.RunAllAsync(publisher.PrepareModuleOutput); + } + } + + /// <summary> + /// Executes test commands for all loaded modules + /// </summary> + /// <returns></returns> + public async Task ExecuteTestsAsync(bool failOnError) + { + foreach (ModuleBase module in _selected) + { + await module.DoRunTests(failOnError); + } + } + + /// <summary> + /// Performs a manual upload step + /// </summary> + /// <returns></returns> + public async Task ManualUpload(BuildPublisher publisher, IUploadManager uploads) + { + //Upload module output + foreach (IModuleData module in _selected) + { + //Upload module + await publisher.UploadModuleOutput(uploads, module); + } + } + + /// <summary> + /// Cleans all modules and child projects + /// </summary> + /// <returns>A task that resolves when all child projects have been cleaned</returns> + public async Task DoStepCleanAsync() + { + //Clean synchronously + foreach (IArtifact module in _selected) + { + await module.CleanAsync(); + } + } + + public void Dispose() + { + foreach (IArtifact module in _allModules) + { + module.Dispose(); + } + + //Cleanup internals + _outdatedModules.Clear(); + } + + } +}
\ No newline at end of file diff --git a/src/BuildPublisher.cs b/src/BuildPublisher.cs new file mode 100644 index 0000000..a2dc2bf --- /dev/null +++ b/src/BuildPublisher.cs @@ -0,0 +1,430 @@ +using System; +using System.IO; +using System.Linq; +using System.Text; +using System.Text.Json; +using System.Threading.Tasks; +using System.Collections.Generic; + +using LibGit2Sharp; + +using Semver; + +using VNLib.Tools.Build.Executor.Constants; +using VNLib.Tools.Build.Executor.Model; +using VNLib.Tools.Build.Executor.Extensions; +using VNLib.Tools.Build.Executor.Projects; +using static VNLib.Tools.Build.Executor.Constants.Config; + +namespace VNLib.Tools.Build.Executor +{ + + public sealed class BuildPublisher(BuildConfig config, GpgSigner signer) + { + public bool SignEnabled => signer.IsEnabled; + + /// <summary> + /// Prepares the module output and its collection of file details for publishing + /// then runs the upload step + /// </summary> + /// <param name="module"></param> + /// <returns>A task that completes when the module's output has been created</returns> + public async Task PrepareModuleOutput(IModuleData module) + { + //Copy project artifacts to output directory + await CopyProjectOutputToModuleOutputAsync(module); + + //Copy source archive + string? archiveFile = await CopySourceArchiveToOutput(module, module.FileManager); + + Log.Information("Building module {mod} catalog and git history", module.ModuleName); + + //Build module catalog + await BuildModuleCatalogAsync(module, config.SemverStyle, archiveFile); + + Log.Information("Building module {mod} git history", module.ModuleName); + + //Build git history + await BuildModuleGitHistoryAsync(module); + + Log.Information("Building module {mod} version history", module.ModuleName); + + //build version history + await BuildModuleVersionHistory(module); + + Log.Information("Moving module {mod} artifacts to the output", module.ModuleName); + } + + /// <summary> + /// Uploads the modules output to the remote + /// </summary> + /// <param name="module">The module containing the information to upload</param> + /// <returns></returns> + public Task UploadModuleOutput(IUploadManager Uploader, IModuleData module) + { + Log.Information("Uploading module {mod}", module.ModuleName); + + //Upload the entire output directory + return Uploader.UploadDirectoryAsync(module.FileManager.OutputDir); + } + + /* + * Builds the project catalog file and publishes it to the module file manager + */ + private async Task BuildModuleCatalogAsync(IModuleData mod, SemVersionStyles style, string? archiveFile) + { + /* + * Builds the index.json file for the module. It + * contains an array of projects and their metadata + */ + + string moduleVersion = mod.GetModuleCiVersion(config.DefaultCiVersion, style); + + using MemoryStream ms = new(); + + using (Utf8JsonWriter writer = new(ms)) + { + //Open initial object + writer.WriteStartObject(); + + InitModuleFile(writer, mod); + + //Add the archive path if it was created in the module + if (archiveFile != null) + { + writer.WriteStartObject("archive"); + + //Archive path is in the build directory + writer.WriteString("path", config.SourceArchiveName); + + //Get the checksum of the archive + string checksum = await new FileInfo(archiveFile).ComputeFileHashStringAsync(); + writer.WriteString(config.HashFuncName, checksum); + writer.WriteString("sha_file", $"{config.SourceArchiveName}.{config.HashFuncName}"); + + //If signing is enabled, add the signature file (it is constant) + if (SignEnabled) + { + writer.WriteString("signature", $"{config.SourceArchiveName}.sig"); + } + + writer.WriteEndObject(); + } + + //Build project array + writer.WriteStartArray("projects"); + + foreach (IProject project in mod.Projects) + { + //start object for each project + writer.WriteStartObject(); + + //Write the project info + await WriteProjectInfoAsync(mod.FileManager, project, mod.Repository.Head.Tip.Sha, moduleVersion, writer); + + writer.WriteEndObject(); + } + + writer.WriteEndArray(); + + //Close object + writer.WriteEndObject(); + + writer.Flush(); + } + + ms.Seek(0, SeekOrigin.Begin); + + await mod.FileManager.WriteFileAsync(ModuleFileType.Catalog, ms.ToArray()); + } + + private static async Task BuildModuleVersionHistory(IModuleData mod) + { + /* + * Builds the index.json file for the module. It + * contains an array of projects and their metadata + */ + + using MemoryStream ms = new(); + + using (Utf8JsonWriter writer = new(ms)) + { + //Open initial object + writer.WriteStartObject(); + + InitModuleFile(writer, mod); + + //Set the head pointer to the latest commit we build + writer.WriteString("head", mod.Repository.Head.Tip.Sha); + + //Build project array + writer.WriteStartArray("versions"); + + //Write all git hashes from head back to the first commit + foreach(Commit commit in mod.Repository.Commits) + { + writer.WriteStringValue(commit.Sha); + } + + writer.WriteEndArray(); + + //Releases will be an array of objects containing the tag and the hash + writer.WriteStartArray("releases"); + + //Write all git tags + foreach(Tag tag in mod.Repository.Tags.OrderByDescending(static p => p.FriendlyName)) + { + writer.WriteStartObject(); + + writer.WriteString("tag", tag.FriendlyName); + writer.WriteString("hash", tag.Target.Sha); + writer.WriteEndObject(); + } + + writer.WriteEndArray(); + + //Close object + writer.WriteEndObject(); + + writer.Flush(); + } + + ms.Seek(0, SeekOrigin.Begin); + + await mod.FileManager.WriteFileAsync(ModuleFileType.VersionHistory, ms.ToArray()); + } + + + /* + * Builds the project's git history file and publishes it to the module file manager + * + * Also updates the latest hash file + */ + private static async Task BuildModuleGitHistoryAsync(IModuleData mod) + { + using MemoryStream ms = new(); + + using (Utf8JsonWriter writer = new(ms)) + { + //Open initial object + writer.WriteStartObject(); + + InitModuleFile(writer, mod); + + //Write the head commit + writer.WriteStartObject("head"); + writer.WriteString("branch", mod.Repository.Head.FriendlyName); + WriteSingleCommit(writer, mod.Repository.Head.Tip); + writer.WriteEndObject(); + + //Write commit history + WriteCommitHistory(writer, mod.Repository); + + //Close object + writer.WriteEndObject(); + + writer.Flush(); + } + + ms.Seek(0, SeekOrigin.Begin); + + await mod.FileManager.WriteFileAsync(ModuleFileType.GitHistory, ms.ToArray()); + + await mod.FileManager.WriteFileAsync(ModuleFileType.LatestHash, Encoding.UTF8.GetBytes(mod.Repository.Head.Tip.Sha)); + } + + /* + * Captures all of the project artiacts and copies them to the module output directory + */ + private async Task CopyProjectOutputToModuleOutputAsync(IModuleData mod) + { + //Copy build artifacts to module output directory + await mod.Projects.RunAllAsync(project => + { + //Get all output files from the project build, and copy them to the module output directory + return project.GetProjectBuildFiles(config) + .RunAllAsync(artifact => mod.FileManager.CopyArtifactToOutputAsync(project, artifact)); + }); + + /* + * If signing is enabled, we can sign all project files synchronously + */ + if (SignEnabled) + { + Log.Information("GPG Siginig is enabled, signing all artifacts for module {mod}", mod.ModuleName); + + /* + * Get all of the artifacts from the module's projects that match the target output + * file type, and sign them + */ + IEnumerable<FileInfo> artifacts = mod.Projects.SelectMany( + p => mod.FileManager.GetArtifactOutputDir(p) + .EnumerateFiles(config.OutputFileType, SearchOption.TopDirectoryOnly) + ); + + //Sign synchronously + foreach(FileInfo artifact in artifacts) + { + await signer.SignFileAsync(artifact); + } + } + } + + + private static void InitModuleFile(Utf8JsonWriter writer, IModuleData mod) + { + //Set object name + writer.WriteString("module_name", mod.ModuleName); + //Modified date + writer.WriteString("modifed_date", DateTime.UtcNow); + } + + private static void WriteCommitHistory(Utf8JsonWriter writer, Repository repo) + { + writer.WriteStartArray("commits"); + + //Write commit history for current repo + foreach (Commit commit in repo.Head.Commits) + { + writer.WriteStartObject(); + + WriteSingleCommit(writer, commit); + + writer.WriteEndObject(); + } + + writer.WriteEndArray(); + + //Write tag history for current repo + writer.WriteStartArray("tags"); + + foreach (Tag tag in repo.Tags) + { + //clamp message length and ellipsis if too long + string? message = tag.Annotation?.Message; + if (message != null && message.Length > 120) + { + message = $"{message[..120]}..."; + } + + writer.WriteStartObject(); + writer.WriteString("name", tag.FriendlyName); + writer.WriteString("sha", tag.Target.Sha); + writer.WriteString("message", message); + writer.WriteString("author", tag.Annotation?.Tagger.Name); + writer.WriteString("date", (tag.Annotation?.Tagger.When ?? default)); + writer.WriteEndObject(); + } + + writer.WriteEndArray(); + } + + private static void WriteSingleCommit(Utf8JsonWriter writer, Commit commit) + { + writer.WriteString("sha", commit.Sha); + writer.WriteString("message", commit.Message); + writer.WriteString("author", commit.Author.Name); + writer.WriteString("commiter", commit.Committer.Name); + writer.WriteString("date", commit.Committer.When); + writer.WriteString("message_short", commit.MessageShort); + } + + /// <summary> + /// Builds and writes the projects information to the <see cref="Utf8JsonWriter"/> + /// </summary> + /// <param name="project"></param> + /// <param name="writer"> + /// The <see cref="Utf8JsonWriter"/> to write the project + /// information to + /// </param> + /// <returns>A task that completes when the write operation has completed</returns> + private async Task WriteProjectInfoAsync(IModuleFileManager man, IProject project, string latestSha, string version, Utf8JsonWriter writer) + { + //Reload the project dom after execute because semversion may be updated after build step + if (project is ModuleProject mp) + { + await mp.LoadProjectDom(); + } + + writer.WriteString("name", project.ProjectName); + writer.WriteString("repo_url", project.ProjectData.RepoUrl); + writer.WriteString("description", project.ProjectData.Description); + writer.WriteString("version", version); + writer.WriteString("copyright", project.ProjectData.Copyright); + writer.WriteString("author", project.ProjectData.Authors); + writer.WriteString("product", project.ProjectData.Product); + writer.WriteString("company", project.ProjectData.CompanyName); + writer.WriteString("commit", latestSha); + //Write target framework if it exsits + writer.WriteString("target_framework", project.ProjectData["TargetFramework"]); + + //Start file array + writer.WriteStartArray("files"); + + //Get only tar files, do not include the sha files + foreach (FileInfo output in GetProjOutputFiles(man, project)) + { + //beging file object + writer.WriteStartObject(); + + writer.WriteString("name", output.Name); + writer.WriteString("path", $"{project.GetSafeProjectName()}/{output.Name}"); + writer.WriteString("date", output.LastWriteTimeUtc); + writer.WriteNumber("size", output.Length); + + //Compute the file hash + string hashHex = await output.ComputeFileHashStringAsync(); + writer.WriteString(config.HashFuncName, hashHex); + + //Path to sha-file + writer.WriteString("sha_file", $"{project.GetSafeProjectName()}/{output.Name}.{config.HashFuncName}"); + + writer.WriteEndObject(); + } + + writer.WriteEndArray(); + } + + private IEnumerable<FileInfo> GetProjOutputFiles(IModuleFileManager man, IProject project) + { + return man.GetArtifactOutputDir(project).EnumerateFiles("*.*", SearchOption.TopDirectoryOnly) + .Where(p => p.Extension != $".{config.HashFuncName}"); + } + + /// <summary> + /// Copies the source git archive tgz file to the output directory + /// </summary> + /// <param name="mod"></param> + /// <param name="man"></param> + /// <returns></returns> + private async Task<string?> CopySourceArchiveToOutput(IModuleData mod, IModuleFileManager man) + { + //Try to get a source archive in the module directory + string? archiveFile = Directory.EnumerateFiles(mod.Repository.Info.WorkingDirectory, config.SourceArchiveName, SearchOption.TopDirectoryOnly).FirstOrDefault(); + + //If archive is null ignore and continue + if(string.IsNullOrWhiteSpace(archiveFile)) + { + Log.Information("No archive file found for module {mod}", mod.ModuleName); + return null; + } + + Log.Information("Found source archive for module {mod}, copying to output...", mod.ModuleName); + + //Otherwise copy to output + byte[] archive = await File.ReadAllBytesAsync(archiveFile); + FileInfo output = await man.WriteFileAsync(ModuleFileType.Archive, archive); + + //Compute the hash of the file + await output.ComputeFileHashAsync(config.HashFuncName); + + if (SignEnabled) + { + //Sign the file if signing is enabled + await signer.SignFileAsync(output); + } + + return archiveFile; + } + } +}
\ No newline at end of file diff --git a/src/BuildStepFailedException.cs b/src/BuildStepFailedException.cs new file mode 100644 index 0000000..0b78bf3 --- /dev/null +++ b/src/BuildStepFailedException.cs @@ -0,0 +1,29 @@ +using System; + +namespace VNLib.Tools.Build.Executor +{ + sealed class BuildStepFailedException : Exception + { + public string? ArtifactName { get; set; } + + public BuildStepFailedException() + { } + + public BuildStepFailedException(string? message) : base(message) + { } + + + public BuildStepFailedException(string? message, Exception? innerException) : base(message, innerException) + { } + + public BuildStepFailedException(string? message, Exception? innerException, string name) : base(message, innerException) + { + ArtifactName = name; + } + + public BuildStepFailedException(string message, string artifactName):base(message) + { + this.ArtifactName = artifactName; + } + } +}
\ No newline at end of file diff --git a/src/Commands/BaseCommand.cs b/src/Commands/BaseCommand.cs new file mode 100644 index 0000000..6362c22 --- /dev/null +++ b/src/Commands/BaseCommand.cs @@ -0,0 +1,116 @@ +using System; +using System.IO; +using System.Threading; +using System.Threading.Tasks; + +using Typin; +using Typin.Console; +using Typin.Attributes; + +using VNLib.Tools.Build.Executor.Model; +using VNLib.Tools.Build.Executor.Constants; +using static VNLib.Tools.Build.Executor.Constants.Config; + +namespace VNLib.Tools.Build.Executor.Commands +{ + public abstract class BaseCommand(BuildPipeline pipeline, ConfigManager bm) : ICommand + { + [CommandOption("verbose", 'v', Description = "Prints verbose output")] + public bool Verbose { get; set; } + + [CommandOption("force", 'f', Description = "Forces the operation even if steps are required")] + public bool Force { get; set; } + + [CommandOption("modules", 'm', Description = "Only use the specified modules, comma separated list")] + public string? Modules { get; set; } + + [CommandOption("exclude", 'x', Description = "Ignores the specified modules, comma separated list")] + public string? Exclude { get; set; } + + [CommandOption("confirm", 'c', Description = "Wait for user input before continuing")] + public bool Confirm { get; set; } + + //Allow users to specify build directory + [CommandOption("build-dir", 'B', Description = "Sets the base build directory to execute operations in")] + public string BuildDir { get; set; } = Directory.GetCurrentDirectory(); + + [CommandOption("max-logs", 'L', Description = "Sets the maximum number of logs to keep")] + public int MaxLogs { get; set; } = 50; + + public IDirectoryIndex BuildIndex { get; private set; } = default!; + + public BuildConfig Config { get; private set; } = default!; + + + /* + * Base exec command does basic init and cleanup on startup + */ + + public virtual async ValueTask ExecuteAsync(IConsole console) + { + try + { + CancellationToken ct = console.GetCancellationToken(); + + //Init build index + BuildIndex = GetIndex(); + Config = await bm.GetOrCreateConfig(BuildIndex, false); + + //Cleanup log files on init + TrimLogs(BuildIndex, MaxLogs); + + string[] modules = Modules?.Split(',') ?? []; + string[] exclude = Exclude?.Split(',') ?? []; + + //Always load the pipeline + await pipeline.LoadAsync(Config, modules, exclude, Feeds); + + if (Confirm) + { + console.Output.WriteLine("---- Pipeline loaded. Press any key to continue ----"); + await console.Input.ReadLineAsync(ct); + + await Task.Delay(100, ct); + ct.ThrowIfCancellationRequested(); + } + + //Exec steps then exit + await ExecStepsAsync(console); + } + catch (OperationCanceledException) + { + console.WithForegroundColor(ConsoleColor.Red, static o => o.Output.WriteLine("Operation cancelled")); + } + catch(BuildStepFailedException be) + { + console.WithForegroundColor(ConsoleColor.Red, o => o.Output.WriteLine("FATAL: Build step failed on module {0}, msg -> {1}", be.ArtifactName, be.Message)); + } + } + + public abstract ValueTask ExecStepsAsync(IConsole console); + + public abstract IFeedManager[] Feeds { get; } + + private Dirs GetIndex() => new() + { + BaseDir = new(BuildDir), + BuildDir = BuildDirs.GetOrCreateDir(BUILD_DIR_NAME), + LogDir = BuildDirs.GetOrCreateDir(LOG_DIR_NAME), + ScratchDir = BuildDirs.GetOrCreateDir(SCRATCH_DIR), + SumDir = BuildDirs.GetOrCreateDir(SUM_DIR), + OutputDir = BuildDirs.GetOrCreateDir(OUTPUT_DIR) + }; + +#nullable disable + protected sealed class Dirs : IDirectoryIndex + { + public DirectoryInfo BaseDir { get; set; } + public DirectoryInfo BuildDir { get; set; } + public DirectoryInfo LogDir { get; set; } + public DirectoryInfo ScratchDir { get; set; } + public DirectoryInfo SumDir { get; set; } + public DirectoryInfo OutputDir { get; set; } + } +#nullable enable + } +}
\ No newline at end of file diff --git a/src/Commands/BuildCommand.cs b/src/Commands/BuildCommand.cs new file mode 100644 index 0000000..c1d9828 --- /dev/null +++ b/src/Commands/BuildCommand.cs @@ -0,0 +1,66 @@ +using System; +using System.Threading; +using System.Threading.Tasks; + +using Typin.Console; +using Typin.Attributes; + +using VNLib.Tools.Build.Executor.Model; +using VNLib.Tools.Build.Executor.Constants; + +namespace VNLib.Tools.Build.Executor.Commands +{ + + [Command("build", Description = "Executes a build operation in pipeline")] + public class BuildCommand(BuildPipeline pipeline, ConfigManager bm) : BaseCommand(pipeline, bm) + { + + [CommandOption("no-delay", 'S', Description = "Skips any built-in delay/wait")] + public bool SkipDelay { get; set; } = false; + + public override async ValueTask ExecStepsAsync(IConsole console) + { + CancellationToken cancellation = console.GetCancellationToken(); + + console.Output.WriteLine("Starting build pipeline. Checking for source code changes"); + + if (Force) + { + console.WithForegroundColor(ConsoleColor.Yellow, static o => o.Output.WriteLine("Forcing build step")); + } + + //Check for source code changes + bool changed = await pipeline.CheckForChangesAsync(); + + //continue build + if (!Force && !changed) + { + console.WithForegroundColor(ConsoleColor.Green, static o => o.Output.WriteLine("No source code changes detected. Skipping build step")); + return; + } + + if (Confirm) + { + console.Output.WriteLine("Press any key to continue..."); + await console.Input.ReadLineAsync(cancellation); + cancellation.ThrowIfCancellationRequested(); + } + else if(!SkipDelay) + { + //wait for 10 seconds + for (int i = 10; i > 0; i--) + { + string seconds = i > 1 ? "seconds" : "second"; + console.Output.WriteLine($"Starting build step in {i} {seconds}"); + await Task.Delay(1000, cancellation); + } + } + + await pipeline.DoStepBuild(Force); + + console.WithForegroundColor(ConsoleColor.Green, static o => o.Output.WriteLine("Build completed successfully")); + } + + public override IFeedManager[] Feeds => []; + } +}
\ No newline at end of file diff --git a/src/Commands/CleanCommand.cs b/src/Commands/CleanCommand.cs new file mode 100644 index 0000000..6570b31 --- /dev/null +++ b/src/Commands/CleanCommand.cs @@ -0,0 +1,31 @@ +using System; +using System.Threading.Tasks; + +using Typin.Console; +using Typin.Attributes; + +using VNLib.Tools.Build.Executor.Model; +using VNLib.Tools.Build.Executor.Constants; + +namespace VNLib.Tools.Build.Executor.Commands +{ + + [Command("clean", Description = "Cleans the build pipeline")] + public sealed class CleanCommand(BuildPipeline pipeline, ConfigManager cfg) : BaseCommand(pipeline, cfg) + { + public override async ValueTask ExecStepsAsync(IConsole console) + { + console.Output.WriteLine("Begining clean step"); + + await pipeline.DoStepCleanAsync(); + + //Cleanup the sum dir and scratch dir + Config.Index.SumDir.Delete(true); + Config.Index.ScratchDir.Delete(true); + + console.WithForegroundColor(ConsoleColor.Green, o => o.Output.WriteLine("Pipeline cleaned")); + } + + public override IFeedManager[] Feeds => []; + } +}
\ No newline at end of file diff --git a/src/Commands/PublishCommand.cs b/src/Commands/PublishCommand.cs new file mode 100644 index 0000000..4179f86 --- /dev/null +++ b/src/Commands/PublishCommand.cs @@ -0,0 +1,80 @@ + +using System; +using System.Linq; +using System.Threading.Tasks; + +using Typin.Console; +using Typin.Attributes; + +using VNLib.Tools.Build.Executor.Model; +using VNLib.Tools.Build.Executor.Constants; + +namespace VNLib.Tools.Build.Executor.Commands +{ + [Command("publish", Description = "Runs publishig build steps on a completed build")] + public sealed class PublishCommand(BuildPipeline pipeline, ConfigManager bm) : BaseCommand(pipeline, bm) + { + [CommandOption("upload-path", 'p', Description = "The path to upload the build artifacts")] + public string? UploadPath { get; set; } + + [CommandOption("sign", 's', Description = "Enables gpg signing of build artifacts")] + public bool Sign { get; set; } = false; + + [CommandOption("gpg-key", 'k', Description = "Optional key to use when signing, otherwise uses the GPG default signing key")] + public string? GpgKey { get; set; } + + [CommandOption("sleet-path", 'F', Description = "Specifies the Sleet feed index path")] + public string? SleetPath { get; set; } + + [CommandOption("dry-run", 'd', Description = "Executes all publish steps without pushing the changes to the remote server")] + public bool DryRun { get; set; } + + [CommandOption("output", 'o', Description = "Specifies the output directory for the published modules")] + public string? CustomOutDir { get; set; } + + public override async ValueTask ExecStepsAsync(IConsole console) + { + //Specify custom output dir + (base.Config.Index as Dirs)!.OutputDir = BuildDirs.GetOrCreateDir(Constants.Config.OUTPUT_DIR, CustomOutDir); + + IUploadManager? uploads = MinioUploadManager.Create(UploadPath); + IFeedManager? feed = Feeds.FirstOrDefault(); + + //Optional gpg signer for signing published artifacts + BuildPublisher pub = new(Config, new GpgSigner(Sign, GpgKey)); + + console.WithForegroundColor(ConsoleColor.DarkGreen, static o => o.Output.WriteLine("Publishing modules")); + + //Run publish steps + await pipeline.OnPublishingAsync().ConfigureAwait(false); + + console.WithForegroundColor(ConsoleColor.DarkGreen, static o => o.Output.WriteLine("Preparing module output for upload")); + + //Prepare the output + await pipeline.PrepareOutputAsync(pub).ConfigureAwait(false); + + if(uploads is null) + { + console.WithForegroundColor(ConsoleColor.DarkYellow, static o => o.Output.WriteLine("No upload path specified. Skipping upload")); + console.WithForegroundColor(ConsoleColor.Green, static o => o.Output.WriteLine("Upload build complete")); + return; + } + + //Run upload + await pipeline.ManualUpload(pub, uploads).ConfigureAwait(false); + + //Publish feeds + if (feed is not null) + { + console.WithForegroundColor(ConsoleColor.DarkGreen, static o => o.Output.WriteLine("Uploading feeds...")); + + //Exec feed upload + await uploads.UploadDirectoryAsync(feed.FeedOutputDir); + } + + console.WithForegroundColor(ConsoleColor.Green, static o => o.Output.WriteLine("Upload build complete")); + } + + public override IFeedManager[] Feeds => SleetPath is null ? [] : [SleetFeedManager.GetSleetFeed(SleetPath)]; + } +}
\ No newline at end of file diff --git a/src/Commands/TestCommand.cs b/src/Commands/TestCommand.cs new file mode 100644 index 0000000..d64fe64 --- /dev/null +++ b/src/Commands/TestCommand.cs @@ -0,0 +1,31 @@ +using System; +using System.Threading.Tasks; + +using Typin.Console; +using Typin.Attributes; + +using VNLib.Tools.Build.Executor.Model; +using VNLib.Tools.Build.Executor.Constants; + +namespace VNLib.Tools.Build.Executor.Commands +{ + [Command("test", Description = "Executes tests steps within the pipline for all loaded modules")] + public sealed class TestCommand(BuildPipeline pipeline, ConfigManager cfg) : BaseCommand(pipeline, cfg) + { + private readonly BuildPipeline _pipeline = pipeline; + + [CommandOption("--no-fail", Description = "Exit testing on the first test failure")] + public bool FailOnTestFail { get; set; } = true; + + public override async ValueTask ExecStepsAsync(IConsole console) + { + console.Output.WriteLine("Begining test step"); + + await _pipeline.ExecuteTestsAsync(FailOnTestFail); + + console.WithForegroundColor(ConsoleColor.Green, o => o.Output.WriteLine("Pipeline tests compled")); + } + + public override IFeedManager[] Feeds => []; + } +}
\ No newline at end of file diff --git a/src/Commands/TestDisplayCommand.cs b/src/Commands/TestDisplayCommand.cs new file mode 100644 index 0000000..c716ada --- /dev/null +++ b/src/Commands/TestDisplayCommand.cs @@ -0,0 +1,22 @@ +using System.Threading.Tasks; + +using Typin.Console; +using Typin.Attributes; + +using VNLib.Tools.Build.Executor.Model; +using VNLib.Tools.Build.Executor.Constants; + +namespace VNLib.Tools.Build.Executor.Commands +{ + [Command("display", Description = "Test command for debugging")] + public sealed class TestDisplayCommand(BuildPipeline pipeline, ConfigManager bm) : BaseCommand(pipeline, bm) + { + public override IFeedManager[] Feeds => []; + + public override async ValueTask ExecStepsAsync(IConsole console) + { + console.Output.WriteLine("Press any key to exit..."); + await console.Input.ReadLineAsync(console.GetCancellationToken()); + } + } +}
\ No newline at end of file diff --git a/src/Commands/UpdateCommand.cs b/src/Commands/UpdateCommand.cs new file mode 100644 index 0000000..e3bdf9c --- /dev/null +++ b/src/Commands/UpdateCommand.cs @@ -0,0 +1,25 @@ +using System; +using System.Threading.Tasks; + +using Typin.Console; +using Typin.Attributes; + +using VNLib.Tools.Build.Executor.Model; +using VNLib.Tools.Build.Executor.Constants; + +namespace VNLib.Tools.Build.Executor.Commands +{ + [Command("update", Description = "Runs the build steps for updating application soure code")] + public sealed class UpdateCommand(BuildPipeline pipeline, ConfigManager cfg) : BaseCommand(pipeline, cfg) + { + public override async ValueTask ExecStepsAsync(IConsole console) + { + //Run the update step + await pipeline.DoStepUpdateSource(); + + console.WithForegroundColor(ConsoleColor.Green, o => o.Output.WriteLine("Source update complete")); + } + + public override IFeedManager[] Feeds => []; + } +}
\ No newline at end of file diff --git a/src/Constants/BuildConfig.cs b/src/Constants/BuildConfig.cs new file mode 100644 index 0000000..39758ea --- /dev/null +++ b/src/Constants/BuildConfig.cs @@ -0,0 +1,77 @@ +using System.Text.Json.Serialization; + +using Semver; + +using VNLib.Tools.Build.Executor.Model; + +namespace VNLib.Tools.Build.Executor.Constants +{ + public sealed class BuildConfig + { + [JsonPropertyName("soure_file_extensions")] + public string[] SourceFileEx { get; set; } = [ + "c", + "cpp", + "cxx", + "h", + "hpp", + "cs", + "proj", + "sln", + "ts", + "js", + "java", + "json", + "yaml", + "yml", + ]; + + [JsonPropertyName("excluded_dirs")] + public string[] ExcludedSourceDirs { get; set; } = [ + "bin", + "obj", + "packages", + "node_modules", + "dist", + "build", + "out", + "target", + ]; + + [JsonPropertyName("default_sha_method")] + public string HashFuncName { get; set; } = "sha256"; + + [JsonPropertyName("head_file_name")] + public string HeadFileName { get; set; } = "@latest"; + + [JsonPropertyName("module_task_file_name")] + public string ModuleTaskFileName { get; set; } = "Module.Taskfile.yaml"; + + [JsonPropertyName("main_taskfile_name")] + public string MainTaskFileName { get; set; } = "build.taskfile.yaml"; + + [JsonPropertyName("output_file_type")] + public string OutputFileType { get; set; } = "*.tgz"; + + [JsonPropertyName("task_exe_name")] + public string TaskExeName { get; set; } = "task"; + + [JsonPropertyName("source_archive_name")] + public string SourceArchiveName { get; set; } = "archive.tgz"; + + [JsonPropertyName("source_archive_format")] + public string SourceArchiveFormat { get; set; } = "tgz"; + + [JsonPropertyName("project_bin_dir")] + public string ProjectBinDir { get; set; } = "bin"; + + [JsonPropertyName("default_ci_version")] + public string DefaultCiVersion { get; set; } = "0.1.0"; + + [JsonPropertyName("semver_style")] + public SemVersionStyles SemverStyle { get; set; } + + [JsonIgnore] + public IDirectoryIndex Index { get; set; } = default!; + } +}
\ No newline at end of file diff --git a/src/Constants/BuildDirs.cs b/src/Constants/BuildDirs.cs new file mode 100644 index 0000000..78609f5 --- /dev/null +++ b/src/Constants/BuildDirs.cs @@ -0,0 +1,35 @@ +using System; +using System.IO; + +namespace VNLib.Tools.Build.Executor.Constants +{ + internal static class BuildDirs + { + + private static DirectoryInfo GetProjectDir() + { + //See if dir was specified on command line + string[] args = Environment.GetCommandLineArgs(); + + //Get the build dir + DirectoryInfo dir = new(args.Length > 1 && Directory.Exists(args[1]) ? args[1] : Directory.GetCurrentDirectory()); + + if (!dir.Exists) + { + dir.Create(); + } + return dir; + } + + public static DirectoryInfo GetOrCreateDir(string @default, string? other = null) + { + //Get the scratch dir + DirectoryInfo logDir = new(Path.Combine(GetProjectDir().FullName, other?? @default)); + if (!logDir.Exists) + { + logDir.Create(); + } + return logDir; + } + } +}
\ No newline at end of file diff --git a/src/Constants/Config.cs b/src/Constants/Config.cs new file mode 100644 index 0000000..fc35928 --- /dev/null +++ b/src/Constants/Config.cs @@ -0,0 +1,95 @@ +using System; +using System.IO; +using System.Linq; + +using Serilog; +using Serilog.Core; + +using VNLib.Tools.Build.Executor.Model; + +namespace VNLib.Tools.Build.Executor.Constants +{ + + internal static class Config + { + + //relative local directores to the project root + public const string BUILD_DIR_NAME = ".build"; + public const string LOG_DIR_NAME = ".build/log"; + public const string BUILD_CONFIG = "build.conf.json"; + public const string SCRATCH_DIR = ".build/scratch"; + public const string SUM_DIR = ".build/sums"; + public const string OUTPUT_DIR = ".build/output"; + public const string SLEET_DIR = ".build/feed"; + + /// <summary> + /// Gets the system wide <see cref="Logger"/> log writer instance + /// </summary> + public static Logger Log { get; } = GetLog(); + + const string Template = "{Message:lj}{NewLine}{Exception}"; + + private static Logger GetLog() + { + string[] args = Environment.GetCommandLineArgs(); + + LoggerConfiguration conf = new(); + + if (args.Contains("-v") || args.Contains("--verbose")) + { + //Check for verbose logging level + conf.MinimumLevel.Verbose(); + } + else if (args.Contains("-d") || args.Contains("--debug")) + { + //Check for debug + conf.MinimumLevel.Debug(); + } + else + { + //Default information level + conf.MinimumLevel.Information(); + } + + //Create a console logger unless the silent flag is set + if (!args.Contains("-s")) + { + conf.WriteTo.Console(outputTemplate: Template); + } + + //Creat the new log file + string logFilePath = Path.Combine(LOG_DIR_NAME, $"{DateTimeOffset.Now.ToUnixTimeSeconds()}-log.txt"); + + //Setup the log file output + conf.WriteTo.File(logFilePath, outputTemplate: Template); + + return conf.CreateLogger(); + } + + /// <summary> + /// Cleans up old build log files, so that only 100 log files remain in the log directory + /// </summary> + public static void TrimLogs(IDirectoryIndex dirIndex, int maxLogs) + { + try + { + //Get all log files in the log directory and cleanup any files after the max log count + FileInfo[] toDelete = dirIndex.LogDir.EnumerateFiles("*.txt", SearchOption.TopDirectoryOnly) + .OrderByDescending(static f => f.LastWriteTimeUtc) + .Skip(maxLogs) + .ToArray(); + + foreach (FileInfo file in toDelete) + { + file.Delete(); + } + + Log.Debug("Cleaned {file} log files", toDelete.Length); + } + catch (Exception ex) + { + Log.Warning(ex, "Failed to cleanup log files"); + } + } + } +}
\ No newline at end of file diff --git a/src/Constants/ConfigManager.cs b/src/Constants/ConfigManager.cs new file mode 100644 index 0000000..3a0b295 --- /dev/null +++ b/src/Constants/ConfigManager.cs @@ -0,0 +1,41 @@ +using System.IO; +using System.Text.Json; +using System.Threading.Tasks; + +using Semver; + +using VNLib.Tools.Build.Executor.Model; + +namespace VNLib.Tools.Build.Executor.Constants +{ + public class ConfigManager(SemVersionStyles semver) + { + public async Task<BuildConfig> GetOrCreateConfig(IDirectoryIndex index, bool overwrite) + { + //Get the config file path + string configFilePath = Path.Combine(index.BuildDir.FullName, Config.BUILD_CONFIG); + + BuildConfig? data = new() + { + SemverStyle = semver + }; + + //If the file doesnt exist or we want to overwrite it + if (!File.Exists(configFilePath) || overwrite) + { + //Create a new config file + await using FileStream fs = File.Create(configFilePath); + await JsonSerializer.SerializeAsync(fs, data); + } + else + { + await using FileStream fs = File.OpenRead(configFilePath); + data = await JsonSerializer.DeserializeAsync<BuildConfig>(fs); + } + + data!.Index = index; + + return data; + } + } +}
\ No newline at end of file diff --git a/src/Constants/ConsoleCancelToken.cs b/src/Constants/ConsoleCancelToken.cs new file mode 100644 index 0000000..cc25feb --- /dev/null +++ b/src/Constants/ConsoleCancelToken.cs @@ -0,0 +1,29 @@ +using System; +using System.Threading; + +namespace VNLib.Tools.Build.Executor.Constants +{ + internal sealed class ConsoleCancelToken : IDisposable + { + private readonly CancellationTokenSource _cts = new(); + + public CancellationToken Token => _cts.Token; + + public ConsoleCancelToken() => Console.CancelKeyPress += OnCancel; + + private void OnCancel(object? sender, ConsoleCancelEventArgs e) + { + _cts.Cancel(); + e.Cancel = true; + } + + public void Dispose() + { + //Unsubscribe from event + Console.CancelKeyPress -= OnCancel; + _cts.Dispose(); + + GC.SuppressFinalize(this); + } + } +}
\ No newline at end of file diff --git a/src/Constants/Utils.cs b/src/Constants/Utils.cs new file mode 100644 index 0000000..6047c35 --- /dev/null +++ b/src/Constants/Utils.cs @@ -0,0 +1,151 @@ +using System; +using System.Threading; +using System.Diagnostics; +using System.Threading.Tasks; +using System.Collections.Generic; +using System.Runtime.CompilerServices; + +using static VNLib.Tools.Build.Executor.Constants.Config; + +namespace VNLib.Tools.Build.Executor.Constants +{ + + internal static class Utils + { + + /// <summary> + /// Runs a process by its name/exe file path, and writes its stdout/stderr to + /// the default build log + /// </summary> + /// <param name="process">The name of the process to run</param> + /// <param name="args">CLI arguments to pass to the process</param> + /// <returns>The process exit code</returns> + public static async Task<int> RunProcessAsync(string process, string? workingDir, string[] args, IReadOnlyDictionary<string, string>? env = null) + { + //Init new console cancellation token + using ConsoleCancelToken ctToken = new(); + + ProcessStartInfo psi = new(process) + { + //Redirect streams + RedirectStandardError = true, + RedirectStandardOutput = true, + CreateNoWindow = true, + //Create a child process, not shell + UseShellExecute = false, + WorkingDirectory = workingDir ?? string.Empty, + }; + + if (env != null) + { + //Add all env variables to process + foreach (KeyValuePair<string, string> kv in env) + { + psi.Environment.Add(kv.Key, kv.Value); + } + } + + //Add arguments + foreach (string arg in args) + { + psi.ArgumentList.Add(arg); + } + + using Process proc = new(); + proc.StartInfo = psi; + + //Start the process + proc.Start(); + + Log.Debug("Starting process {proc}, with args {args}", proc.ProcessName, args); + Console.WriteLine(); + + //Log std out + Task stdout = LogStdOutAsync(proc, ctToken.Token); + Task stdErr = LogStdErrAsync(proc, ctToken.Token); + + //Wait for the process to exit + Task wfe = proc.WaitForExitAsync(ctToken.Token); + + //Wait for stderr/out/proc to exit + await Task.WhenAll(stdout, stdErr, wfe); + + Console.WriteLine(); + Log.Debug("[CHILD]:{id}:{p} exited w/ code {code}", proc.ProcessName, proc.Id, proc.ExitCode); + + //Return status code + return proc.ExitCode; + } + + private static async Task LogStdOutAsync(Process psi, CancellationToken cancellation) + { + try + { + string procName = psi.ProcessName; + int id = psi.Id; + + do + { + //Read lines from the process + string? line = await psi.StandardOutput.ReadLineAsync(cancellation); + + if (line == null) + { + break; + } + + //Print to log file + Console.WriteLine(line); + } while (!psi.HasExited); + } + catch (Exception ex) + { + Log.Error(ex, "An exception was raised while reading the process standard output"); + } + } + + private static async Task LogStdErrAsync(Process psi, CancellationToken cancellation) + { + try + { + string procName = psi.ProcessName; + int id = psi.Id; + + do + { + //Read lines from the process + string? line = await psi.StandardError.ReadLineAsync(cancellation); + + if (line == null) + { + break; + } + + //Print to log file + Console.WriteLine(line); + } while (!psi.HasExited); + } + catch (Exception ex) + { + Log.Error(ex, "An exception was raised while reading the process standard output"); + } + } + + + /// <summary> + /// Throws a <see cref="BuildStepFailedException"/> if the value + /// of <paramref name="status"/> is false + /// </summary> + /// <param name="status">If false throws exception</param> + /// <param name="message">The message to display</param> + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static void ThrowIfStepFailed(bool status, string message, string artifactName) + { + if (!status) + { + throw new BuildStepFailedException(message, artifactName); + } + } + + } +}
\ No newline at end of file diff --git a/src/Extensions/BuildExtensions.cs b/src/Extensions/BuildExtensions.cs new file mode 100644 index 0000000..24af05e --- /dev/null +++ b/src/Extensions/BuildExtensions.cs @@ -0,0 +1,150 @@ +using System; +using System.IO; +using System.Linq; +using System.Threading.Tasks; +using System.Collections.Generic; +using System.Security.Cryptography; + +using LibGit2Sharp; + +using Semver; + +using VNLib.Tools.Build.Executor.Model; + +namespace VNLib.Tools.Build.Executor.Extensions +{ + internal static class BuildExtensions + { + + /// <summary> + /// Determines if the file exists within the current directory + /// </summary> + /// <param name="dir"></param> + /// <param name="fileName">The name of the file to search for</param> + /// <returns>True if the file exists, false otherwise</returns> + public static bool FileExists(this DirectoryInfo dir, string fileName) => File.Exists(Path.Combine(dir.FullName, fileName)); + + /// <summary> + /// Determines if a child directory exists + /// </summary> + /// <param name="dir"></param> + /// <param name="dirName">The name of the directory to check for</param> + /// <returns>True if the directory exists</returns> + public static bool ChildExists(this DirectoryInfo dir, string dirName) => Directory.Exists(Path.Combine(dir.FullName, dirName)); + + /// <summary> + /// Deletes a child directory + /// </summary> + /// <param name="dir"></param> + /// <param name="dirName">The name of the directory to delete</param> + /// <param name="recurse">Recursive delete, delete all child items</param> + public static void DeleteChild(this DirectoryInfo dir, string dirName, bool recurse = true) => Directory.Delete(Path.Combine(dir.FullName, dirName), recurse); + + /// <summary> + /// Creates a child directory within the current directory + /// </summary> + /// <param name="dir"></param> + /// <param name="name">The name of the child directory</param> + public static DirectoryInfo CreateChild(this DirectoryInfo dir, string name) => Directory.CreateDirectory(Path.Combine(dir.FullName, name)); + + /// <summary> + /// Computes the SHA256 hash of the current file and writes the hash to + /// a filename.sha256 hexadecimal text file + /// </summary> + /// <param name="file"></param> + /// <returns>A task the completes when the file hash has been produced in the output directory</returns> + public static async Task ComputeFileHashAsync(this FileInfo file, string hashName) + { + string outputName = $"{file.FullName}.{hashName}"; + + //convert the hash to hexadecimal + string hex = await ComputeFileHashStringAsync(file); + + //Write the hex hash to the output file + await File.WriteAllTextAsync(outputName, hex); + } + + /// <summary> + /// Computes the SHA256 hash of the current file and returns the file hash as a hexadecimal string + /// </summary> + /// <param name="file"></param> + /// <returns>A task the completes when the file hash has been produced in the output directory</returns> + public static async Task<string> ComputeFileHashStringAsync(this FileInfo file) + { + using SHA256 alg = SHA256.Create(); + + //Open the output file to read the file data to compute hash + await using FileStream input = file.OpenRead(); + + //Compute hash + byte[] hash = await alg.ComputeHashAsync(input); + + //convert the hash to hexadecimal + return Convert.ToHexString(hash); + } + + + public static Task RunAllAsync<T>(this IEnumerable<T> workCol, Func<T, Task> cb) + { + Task[] tasks = workCol.Select(cb).ToArray(); + return Task.WhenAll(tasks); + } + + /// <summary> + /// Gets the module's version based on the latest tag and the number of commits since the last tag + /// that supports pre-release/semver + /// </summary> + /// <param name="mod"></param> + /// <param name="style">The version style</param> + /// <returns>The ci build/version number</returns> + public static string GetModuleCiVersion(this IModuleData mod, string defaultCiVersion, SemVersionStyles style) + { + int ciNumber = 0; + SemVersion baseVersion; + + //Get latest version tag from git + Tag? vTag = mod.Repository.Tags.OrderByDescending(p => SemVersion.Parse(p.FriendlyName, style)).FirstOrDefault(); + + //Find the number of commits since the last tag + if (vTag != null) + { + //Get the number of commits since the last tag + baseVersion = SemVersion.Parse(vTag.FriendlyName, style); + + //Search through commits till we can find the commit that matches the tag + Commit[] commits = mod.Repository.Commits.ToArray(); + + for (; ciNumber < commits.Length; ciNumber++) + { + if (commits[ciNumber].Sha == vTag.Target.Sha) + { + break; + } + } + } + else + { + //No tags, so just use the number of commits + ciNumber = mod.Repository.Commits.Count() - 1; + baseVersion = SemVersion.Parse(defaultCiVersion, style); + } + + //If there are commits, then increment the version prerelease + if (ciNumber > 0) + { + //Increment the version + baseVersion = baseVersion.WithPrerelease($"ci{ciNumber:0000}"); + } + + return baseVersion.ToString(); + } + + public static bool IsFileIgnored(this Repository repo, string file) + { + FileStatus status = repo.RetrieveStatus(file); + + //If the leaf project is ignored, skip it + return status.HasFlag(FileStatus.Ignored) || status.HasFlag(FileStatus.Nonexistent); + } + } +}
\ No newline at end of file diff --git a/src/Extensions/FileManagerExtensions.cs b/src/Extensions/FileManagerExtensions.cs new file mode 100644 index 0000000..6f497e7 --- /dev/null +++ b/src/Extensions/FileManagerExtensions.cs @@ -0,0 +1,135 @@ +using System; +using System.IO; +using System.Linq; +using System.Text.Json; +using System.Threading.Tasks; +using System.Collections.Generic; +using System.Runtime.CompilerServices; + +using VNLib.Tools.Build.Executor.Model; +using VNLib.Tools.Build.Executor.Constants; +using static VNLib.Tools.Build.Executor.Constants.Config; + +namespace VNLib.Tools.Build.Executor.Extensions +{ + + internal static class FileManagerExtensions + { + + private static readonly ConditionalWeakTable<IProject, object> _projectHashes = new(); + + /// <summary> + /// Gets all external dependencies for the current module + /// </summary> + /// <param name="module"></param> + /// <returns>An array of project names of all the dependencies outside a given module</returns> + public static string[] GetExternalDependencies(this IModuleData module) + { + /* + * We need to get all child project dependencies that rely on projects + * outside of the current module. + * + * This assumes all projects within this model are properly linked + * and assumed to be build together, as is 99% of the case, otherwise + * custom build impl will happen at the Task level + */ + + //Get the project file names contained in the current module + string[] selfProjects = module.Projects.Select(static p => p.ProjectFile.Name).ToArray(); + + return module.Projects.SelectMany(static p => p.GetDependencies()).Where(dep => !selfProjects.Contains(dep)).ToArray(); + } + + /// <summary> + /// Determines if any source files have changed + /// </summary> + /// <param name="commit">The most recent commit hash</param> + public static async Task CheckSourceChangedAsync(this IModuleFileManager manager, IProject project, BuildConfig config, string commit) + { + //Compute current sum + string sum = await project.GetSourceFileHashAsync(config); + + //Old sum file exists + byte[]? sumData = await manager.ReadCheckSumAsync(project); + + //Try to read the old sum file + if (sumData != null) + { + //Parse sum file + using JsonDocument sumDoc = JsonDocument.Parse(sumData); + + //Get the sum + string? hexSum = sumDoc.RootElement.GetProperty("sum").GetString(); + + //Confirm the current sum and the found sum are equal + if (sum.Equals(hexSum, StringComparison.OrdinalIgnoreCase)) + { + Log.Verbose("Project {p} source is {up}", project.ProjectName, "up-to-date"); + + //Project source is up-to-date + project.UpToDate = true; + + //No changes made + return; + } + } + + Log.Verbose("Project {p} source is {up}", project.ProjectName, "changed"); + + //Store sum change + object sumChange = new Dictionary<string, string>() + { + { "sum", sum }, + { "commit", commit }, + { "modified", DateTimeOffset.UtcNow.ToString("s") } + }; + + //Store sum change for later + _projectHashes.Add(project, sumChange); + + project.UpToDate = false; + } + + /// <summary> + /// Creates the module's output directory + /// </summary> + /// <param name="manager"></param> + public static void CreateOutput(this IModuleFileManager manager) + { + //Create output directory for solution + _ = Directory.CreateDirectory(manager.OutputDir); + } + + /// <summary> + /// Deletes the output's module directory + /// </summary> + /// <param name="manager"></param> + public static void CleanOutput(this IModuleFileManager manager) + { + //Delete output directory for solution + if (Directory.Exists(manager.OutputDir)) + { + Directory.Delete(manager.OutputDir, true); + } + } + + + /// <summary> + /// Writes the source file checksum change to the project's sum file + /// </summary> + /// <param name="project"></param> + /// <param name="project">The project to write the checksum for</param> + /// <returns>A task that resolves when the sum file has been updated</returns> + public static Task CommitSumChangeAsync(this IModuleFileManager manager, IProject project) + { + if (!_projectHashes.TryGetValue(project, out object? sumChange)) + { + return Task.CompletedTask; + } + + byte[] sumData = JsonSerializer.SerializeToUtf8Bytes(sumChange); + + return manager.WriteChecksumAsync(project, sumData); + } + } +}
\ No newline at end of file diff --git a/src/Extensions/ProjectExtensions.cs b/src/Extensions/ProjectExtensions.cs new file mode 100644 index 0000000..a24e6c5 --- /dev/null +++ b/src/Extensions/ProjectExtensions.cs @@ -0,0 +1,128 @@ + +using System; +using System.IO; +using System.Linq; +using System.Threading.Tasks; +using System.Collections.Generic; +using System.Security.Cryptography; + +using VNLib.Tools.Build.Executor.Model; +using VNLib.Tools.Build.Executor.Constants; + +namespace VNLib.Tools.Build.Executor.Extensions +{ + + internal static class ProjectExtensions + { + + /// <summary> + /// Gets the project dependencies for the given project + /// </summary> + /// <param name="project"></param> + /// <returns>The list of project dependencies</returns> + public static string[] GetDependencies(this IProject project) + { + //Get the project file names (not paths) that are dependencies + return project.ProjectData.GetProjectRefs().Select(static r => Path.GetFileName(r)).ToArray(); + } + + private static bool IsSourceFile(BuildConfig conf, string fileName) + { + for (int i = 0; i < conf.SourceFileEx.Length; i++) + { + if (fileName.EndsWith(conf.SourceFileEx[i])) + { + return true; + } + } + return false; + } + + private static bool IsExcludedDir(BuildConfig conf, string path) + { + for (int i = 0; i < conf.ExcludedSourceDirs.Length; i++) + { + if (path.Contains(conf.ExcludedSourceDirs[i])) + { + return true; + } + } + return false; + } + + public static IEnumerable<FileInfo> GetProjectBuildFiles(this IProject project, BuildConfig config) + { + //See if an output dir is specified + string? outDir = project.ProjectData["output_dir"] ?? project.ProjectData["output"]; + + //If an output dir is specified, only get files from that dir + if(!string.IsNullOrWhiteSpace(outDir)) + { + //realtive file path + outDir = Path.Combine(project.WorkingDir.FullName, outDir); + + return new DirectoryInfo(outDir).EnumerateFiles(config.OutputFileType, SearchOption.TopDirectoryOnly); + } + else + { + return project.WorkingDir.EnumerateFiles(config.OutputFileType, SearchOption.AllDirectories); + } + } + + /// <summary> + /// Gets the sha256 hash of all the source files within the project + /// </summary> + /// <returns>A task that resolves the hexadecimal string of the sha256 hash of all the project source files</returns> + public static async Task<string> GetSourceFileHashAsync(this IProject project, BuildConfig config) + { + //Get all + FileInfo[] sourceFiles = project.WorkingDir!.EnumerateFiles("*.*", SearchOption.AllDirectories) + //Get all source files, c/c#/c++ source files, along with .xproj files (project files) + .Where(n => IsSourceFile(config, n.Name)) + //Exclude the obj intermediate output dir + .Where(f => !IsExcludedDir(config, f.DirectoryName ?? "")) + .ToArray(); + + //Get a scratch file to append file source code to + await using FileStream scratch = new( + $"{config.Index.ScratchDir.FullName}/{Path.GetRandomFileName()}", + FileMode.OpenOrCreate, + FileAccess.ReadWrite, + FileShare.None, + 8192, + FileOptions.DeleteOnClose + ); + + //Itterate over all source files + foreach (FileInfo sourceFile in sourceFiles) + { + //Open the source file stream + await using FileStream source = sourceFile.OpenRead(); + + //Append the data to the file stream + await source.CopyToAsync(scratch); + } + + //Flush the stream to disk and roll back to start + await scratch.FlushAsync(); + scratch.Seek(0, SeekOrigin.Begin); + + byte[] hash; + + //Create a sha 256 hash of the file + using (SHA256 alg = SHA256.Create()) + { + //Hash the file + hash = await alg.ComputeHashAsync(scratch); + } + + //Get hex of the hash + return Convert.ToHexString(hash); + } + + public static string GetSafeProjectName(this IProject project) + { + return project.ProjectName.Replace('/', '-').Replace('\\','-'); + } + } +}
\ No newline at end of file diff --git a/src/GpgSigner.cs b/src/GpgSigner.cs new file mode 100644 index 0000000..76943f9 --- /dev/null +++ b/src/GpgSigner.cs @@ -0,0 +1,52 @@ +using System; +using System.IO; +using System.Threading.Tasks; +using System.Collections.Generic; + +using VNLib.Tools.Build.Executor.Constants; + +namespace VNLib.Tools.Build.Executor +{ + public sealed class GpgSigner(bool enabled, string? defaultKey) + { + public bool IsEnabled { get; } = enabled; + + public async Task SignFileAsync(FileInfo file) + { + if (!IsEnabled) + { + return; + } + + List<string> args = [ "--detach-sign" ]; + + if (!string.IsNullOrWhiteSpace(defaultKey)) + { + //Set the preferred key + args.Add("--default-key"); + args.Add(defaultKey); + } + + //Add input file + args.Add(file.FullName); + + //Delete an original file + string sigFile = $"{file.FullName}.sig"; + if (File.Exists(sigFile)) + { + File.Delete(sigFile); + } + + int result = await Utils.RunProcessAsync("gpg", null, args.ToArray()); + + switch (result) + { + case 2: + case 0: + break; + default: + throw new Exception($"Failed to sign file {file.FullName}"); + } + } + } +}
\ No newline at end of file diff --git a/src/MinioUploadManager.cs b/src/MinioUploadManager.cs new file mode 100644 index 0000000..8337438 --- /dev/null +++ b/src/MinioUploadManager.cs @@ -0,0 +1,60 @@ +using System.Diagnostics.CodeAnalysis; +using System.Threading.Tasks; + +using VNLib.Tools.Build.Executor.Model; + +using static VNLib.Tools.Build.Executor.Constants.Utils; + +namespace VNLib.Tools.Build.Executor +{ + internal class MinioUploadManager : IUploadManager + { + private readonly string _minioPath; + + private MinioUploadManager(string minioPath) + { + _minioPath = minioPath; + } + + public Task CleanAllAsync(string path) + { + throw new System.NotImplementedException(); + } + + public Task DeleteFileAsync(string filePath) + { + throw new System.NotImplementedException(); + } + + public async Task UploadDirectoryAsync(string path) + { + //Recursivley copy all files in the working directory + string[] args = + { + "cp", + "--recursive", + ".", + _minioPath + }; + + //Set working dir to the supplied dir path, and run the command + int result = await RunProcessAsync("mc", path, args); + + if(result != 0) + { + throw new BuildStepFailedException($"Failed to upload directory {path} with status code {result:x}"); + } + } + + public Task UploadFileAsync(string filePath) + { + throw new System.NotImplementedException(); + } + + [return:NotNullIfNotNull(nameof(uploadPath))] + public static IUploadManager? Create(string? uploadPath) + { + return string.IsNullOrWhiteSpace(uploadPath) ? null : new MinioUploadManager(uploadPath); + } + } +}
\ No newline at end of file diff --git a/src/Model/IArtifact.cs b/src/Model/IArtifact.cs new file mode 100644 index 0000000..c3c215a --- /dev/null +++ b/src/Model/IArtifact.cs @@ -0,0 +1,23 @@ +using System; +using System.Threading.Tasks; + +namespace VNLib.Tools.Build.Executor.Model +{ + internal interface IArtifact : IDisposable + { + /// <summary> + /// Invoked when the executor requests all created artifacts load async assets + /// and update state accordingly + /// </summary> + /// <param name="vars">The taskfile variable container</param> + /// <returns>A task that completes when all assets are loaded</returns> + Task LoadAsync(TaskfileVars vars); + + /// <summary> + /// Invoked when the executor requests all artifacts cleanup assets that + /// may have been generated during a build process + /// </summary> + /// <returns>A task that completes when all assest are cleaned</returns> + Task CleanAsync(); + } +}
\ No newline at end of file diff --git a/src/Model/IBuildable.cs b/src/Model/IBuildable.cs new file mode 100644 index 0000000..dd32e03 --- /dev/null +++ b/src/Model/IBuildable.cs @@ -0,0 +1,19 @@ +using System.Threading.Tasks; + +namespace VNLib.Tools.Build.Executor.Model +{ + internal interface IBuildable : IArtifact + { + Task DoStepSyncSource(); + + Task<bool> CheckForChangesAsync(); + + Task DoStepBuild(); + + Task DoStepPostBuild(bool success); + + Task DoStepPublish(); + + Task DoRunTests(bool failOnError); + } +}
\ No newline at end of file diff --git a/src/Model/IDirectoryIndex.cs b/src/Model/IDirectoryIndex.cs new file mode 100644 index 0000000..4a30344 --- /dev/null +++ b/src/Model/IDirectoryIndex.cs @@ -0,0 +1,37 @@ +using System.IO; + +namespace VNLib.Tools.Build.Executor.Model +{ + public interface IDirectoryIndex + { + /// <summary> + /// The current build base directory + /// </summary> + DirectoryInfo BaseDir { get; } + + /// <summary> + /// The top level internal build directory + /// </summary> + DirectoryInfo BuildDir { get; } + + /// <summary> + /// The directory where log files are stored + /// </summary> + DirectoryInfo LogDir { get; } + + /// <summary> + /// Gets the build scratch directory + /// </summary> + DirectoryInfo ScratchDir { get; } + + /// <summary> + /// Gets the build checksum directory, used to store source file sums + /// </summary> + DirectoryInfo SumDir { get; } + + /// <summary> + /// The build output directory + /// </summary> + DirectoryInfo OutputDir { get; } + } +}
\ No newline at end of file diff --git a/src/Model/IFeedManager.cs b/src/Model/IFeedManager.cs new file mode 100644 index 0000000..7f56449 --- /dev/null +++ b/src/Model/IFeedManager.cs @@ -0,0 +1,16 @@ +namespace VNLib.Tools.Build.Executor.Model +{ + public interface IFeedManager + { + /// <summary> + /// Adds taskfile variables for the feed manager + /// </summary> + /// <param name="vars">The taskfile variable container</param> + void AddVariables(TaskfileVars vars); + + /// <summary> + /// The output directory of the feed + /// </summary> + string FeedOutputDir { get; } + } +}
\ No newline at end of file diff --git a/src/Model/IModuleData.cs b/src/Model/IModuleData.cs new file mode 100644 index 0000000..c41a76b --- /dev/null +++ b/src/Model/IModuleData.cs @@ -0,0 +1,19 @@ +using System.Collections.Generic; + +using LibGit2Sharp; + +namespace VNLib.Tools.Build.Executor.Model +{ + public interface IModuleData + { + ICollection<IProject> Projects { get; } + + string ModuleName { get; } + + Repository Repository { get; } + + TaskfileVars TaskVars { get; } + + IModuleFileManager FileManager { get; } + } +}
\ No newline at end of file diff --git a/src/Model/IModuleFileManager.cs b/src/Model/IModuleFileManager.cs new file mode 100644 index 0000000..cf1a325 --- /dev/null +++ b/src/Model/IModuleFileManager.cs @@ -0,0 +1,62 @@ +using System.IO; +using System.Threading.Tasks; + +namespace VNLib.Tools.Build.Executor.Model +{ + public enum ModuleFileType + { + None, + Catalog, + GitHistory, + Checksum, + LatestHash, + VersionHistory, + Archive + } + + public interface IModuleFileManager + { + /// <summary> + /// Writes the file to the module output directory + /// </summary> + /// <param name="type">The <see cref="ModuleFileType"/> to write</param> + /// <param name="fileData">The file data to write</param> + /// <returns>A task that resolves when the file has been written</returns> + Task<FileInfo> WriteFileAsync(ModuleFileType type, byte[] fileData); + + /// <summary> + /// Writes the checksum file to the sum's output directory for the given project + /// </summary> + /// <param name="project">The project to write the sum file data for</param> + /// <param name="fileData"></param> + /// <returns></returns> + Task WriteChecksumAsync(IProject project, byte[] fileData); + + /// <summary> + /// Attemts to read the checksum file data for the given project + /// </summary> + /// <param name="project">The project to get the sum data for</param> + /// <returns>The file contents of the sum file, or null if the file does not exist</returns> + Task<byte[]?> ReadCheckSumAsync(IProject project); + + /// <summary> + /// The module's output directory + /// </summary> + string OutputDir { get; } + + /// <summary> + /// Copies the given file to the project's output directory + /// </summary> + /// <param name="project"></param> + /// <param name="file"></param> + /// <returns></returns> + Task CopyArtifactToOutputAsync(IProject project, FileInfo file); + + /// <summary> + /// Gets the output directory for the given project + /// </summary> + /// <param name="project">The project to get the artifact output of</param> + /// <returns>A <see cref="DirectoryInfo"/> object describing the output dir</returns> + DirectoryInfo GetArtifactOutputDir(IProject project); + } +}
\ No newline at end of file diff --git a/src/Model/IProject.cs b/src/Model/IProject.cs new file mode 100644 index 0000000..e7f15bb --- /dev/null +++ b/src/Model/IProject.cs @@ -0,0 +1,37 @@ +using System.IO; +using System.Threading.Tasks; + +namespace VNLib.Tools.Build.Executor.Model +{ + public interface IProject : ITaskfileScope + { + /// <summary> + /// Gets the the project file + /// </summary> + FileInfo ProjectFile { get; } + + /// <summary> + /// Gets the actual project name + /// </summary> + string ProjectName { get; } + + /// <summary> + /// The msbuild project dom + /// </summary> + IProjectData ProjectData { get; } + + /// <summary> + /// A value that indicates (after a source sync) that the project + /// is considered up to date. + /// </summary> + bool UpToDate { get; set; } + + /// <summary> + /// Invoked when the executor requests all created artifacts load async assets + /// and update state accordingly + /// </summary> + /// <param name="vars">The taskfile variable container</param> + /// <returns>A task that completes when all assets are loaded</returns> + Task LoadAsync(TaskfileVars vars); + } +}
\ No newline at end of file diff --git a/src/Model/IProjectData.cs b/src/Model/IProjectData.cs new file mode 100644 index 0000000..1b4190d --- /dev/null +++ b/src/Model/IProjectData.cs @@ -0,0 +1,19 @@ +using System.IO; + + +namespace VNLib.Tools.Build.Executor.Model +{ + public interface IProjectData + { + string? Description { get; } + string? Authors { get; } + string? Copyright { get; } + string? VersionString { get; } + string? CompanyName { get; } + string? Product { get; } + string? RepoUrl { get; } + string? this[string index] { get; } + void Load(Stream stream); + string[] GetProjectRefs(); + } +}
\ No newline at end of file diff --git a/src/Model/IProjectExplorer.cs b/src/Model/IProjectExplorer.cs new file mode 100644 index 0000000..3c9c61b --- /dev/null +++ b/src/Model/IProjectExplorer.cs @@ -0,0 +1,16 @@ +using System.Collections.Generic; + +namespace VNLib.Tools.Build.Executor.Model +{ + /// <summary> + /// Represents a project explorer, capable of discovering projects within a module + /// </summary> + internal interface IProjectExplorer + { + /// <summary> + /// Discovers all projects within the module + /// </summary> + /// <returns>An enumeration of projects discovered</returns> + IEnumerable<IProject> DiscoverProjects(); + } +}
\ No newline at end of file diff --git a/src/Model/ITaskfileScope.cs b/src/Model/ITaskfileScope.cs new file mode 100644 index 0000000..6a416fa --- /dev/null +++ b/src/Model/ITaskfileScope.cs @@ -0,0 +1,22 @@ +using System.IO; + +namespace VNLib.Tools.Build.Executor.Model +{ + public interface ITaskfileScope + { + /// <summary> + /// The taskfile working directory + /// </summary> + DirectoryInfo WorkingDir { get; } + + /// <summary> + /// The taskfile variable container + /// </summary> + TaskfileVars TaskVars { get; } + + /// <summary> + /// The optional taskfile name + /// </summary> + string? TaskfileName { get; } + } +}
\ No newline at end of file diff --git a/src/Model/IUploadManager.cs b/src/Model/IUploadManager.cs new file mode 100644 index 0000000..01d7081 --- /dev/null +++ b/src/Model/IUploadManager.cs @@ -0,0 +1,15 @@ +using System.Threading.Tasks; + +namespace VNLib.Tools.Build.Executor.Model +{ + public interface IUploadManager + { + Task CleanAllAsync(string path); + + Task DeleteFileAsync(string filePath); + + Task UploadDirectoryAsync(string path); + + Task UploadFileAsync(string filePath); + } +}
\ No newline at end of file diff --git a/src/Model/TaskfileVars.cs b/src/Model/TaskfileVars.cs new file mode 100644 index 0000000..c0e6cdc --- /dev/null +++ b/src/Model/TaskfileVars.cs @@ -0,0 +1,48 @@ +using System; +using System.Collections.Generic; + +namespace VNLib.Tools.Build.Executor.Model +{ + /// <summary> + /// Represents a collection of taskfile "environment" variables + /// </summary> + public sealed class TaskfileVars + { + private readonly Dictionary<string, string> vars; + + public TaskfileVars() + { + vars = new(StringComparer.OrdinalIgnoreCase); + } + + private TaskfileVars(IEnumerable<KeyValuePair<string, string>> values) + { + vars = new(values, StringComparer.OrdinalIgnoreCase); + } + + /// <summary> + /// Gets all variables as a readonly dictionary + /// </summary> + /// <returns>The collection of environment variables</returns> + public IReadOnlyDictionary<string, string> GetVariables() => vars; + + /// <summary> + /// Sets a taskfile environment variable + /// </summary> + /// <param name="key">The variable name</param> + /// <param name="value">The optional variable value</param> + public void Set(string key, string? value) => vars[key] = value ?? string.Empty; + + /// <summary> + /// Removes a taskfile environment variable + /// </summary> + /// <param name="key">The name of the variable to remove</param> + public void Remove(string key) => vars.Remove(key); + + /// <summary> + /// Clones the current taskfile variables into an independent instance + /// </summary> + /// <returns>The new <see cref="TaskfileVars"/> instance</returns> + public TaskfileVars Clone() => new (vars); + } +}
\ No newline at end of file diff --git a/src/Modules/GitCodeModule.cs b/src/Modules/GitCodeModule.cs new file mode 100644 index 0000000..630922d --- /dev/null +++ b/src/Modules/GitCodeModule.cs @@ -0,0 +1,42 @@ +using System.IO; +using System.Linq; +using System.Threading.Tasks; + +using VNLib.Tools.Build.Executor.Constants; +using VNLib.Tools.Build.Executor.Model; + +namespace VNLib.Tools.Build.Executor.Modules +{ + internal sealed class GitCodeModule : ModuleBase + { + private string _moduleName; + + public override IProjectExplorer ModuleExplorer { get; } + + public override string ModuleName => _moduleName; + + public GitCodeModule(BuildConfig config, DirectoryInfo root) + : base(config, root) + { + ModuleExplorer = new MsBuildModuleExplorer(config, this, root); + _moduleName = root.Name; //Default to dir name + } + + ///<inheritdoc/> + public override Task LoadAsync(TaskfileVars vars) + { + //Try to load a solution file in the top-level module directory + FileInfo? sln = WorkingDir.EnumerateFiles("*.sln", SearchOption.TopDirectoryOnly).FirstOrDefault(); + + if(sln is not null) + { + //Remove the .build extension from the solution file name for proper module name + _moduleName = Path.GetFileNameWithoutExtension(sln.Name).Replace(".build", string.Empty); + + vars.Set("SOLUTION_FILE_NAME", sln.Name); + } + + return base.LoadAsync(vars); + } + } +}
\ No newline at end of file diff --git a/src/Modules/ModuleBase.cs b/src/Modules/ModuleBase.cs new file mode 100644 index 0000000..750cf61 --- /dev/null +++ b/src/Modules/ModuleBase.cs @@ -0,0 +1,258 @@ +using System; +using System.IO; +using System.Linq; +using System.Threading.Tasks; +using System.Collections.Generic; + +using LibGit2Sharp; + +using VNLib.Tools.Build.Executor.Model; +using VNLib.Tools.Build.Executor.Constants; +using VNLib.Tools.Build.Executor.Extensions; +using static VNLib.Tools.Build.Executor.Constants.Config; + +namespace VNLib.Tools.Build.Executor.Modules +{ + /// <summary> + /// Represents a base class for all modules to inherit from + /// </summary> + internal abstract class ModuleBase : IArtifact, IBuildable, IModuleData, ITaskfileScope + { + protected readonly BuildConfig Config; + protected readonly TaskFile TaskFile; + + ///<inheritdoc/> + public TaskfileVars TaskVars { get; private set; } + + ///<inheritdoc/> + public DirectoryInfo WorkingDir { get; } + + ///<inheritdoc/> + public abstract string ModuleName { get; } + + ///<inheritdoc/> + public ICollection<IProject> Projects { get; } = new LinkedList<IProject>(); + + ///<inheritdoc/> + public IModuleFileManager FileManager { get; } + + ///<inheritdoc/> + public string? TaskfileName { get; protected set; } + + /// <summary> + /// The git repository of the module + /// </summary> + public Repository Repository { get; } + + /// <summary> + /// The project explorer for the module + /// </summary> + public abstract IProjectExplorer ModuleExplorer { get; } + + public ModuleBase(BuildConfig config, DirectoryInfo root) + { + WorkingDir = root; + Config = config; + + //Default to module taskfile name + TaskfileName = config.ModuleTaskFileName; + + //Init repo for root working dir + Repository = new(root.FullName); + FileManager = new ModuleFileManager(config, this); + TaskVars = null!; + + TaskFile = new(config.TaskExeName, () => ModuleName); + } + + ///<inheritdoc/> + public virtual async Task LoadAsync(TaskfileVars vars) + { + if(Repository?.Head?.Tip?.Sha is null) + { + throw new BuildStepFailedException("This repository does not have any commit history. Cannot continue"); + } + + //Store paraent vars + TaskVars = vars; + + string moduleSemVer = this.GetModuleCiVersion(Config.DefaultCiVersion, Config.SemverStyle); + + //Build module local environment variables + TaskVars.Set("MODULE_NAME", ModuleName); + TaskVars.Set("OUTPUT_DIR", FileManager.OutputDir); + TaskVars.Set("MODULE_DIR", WorkingDir.FullName); + + //Store current head-sha before update step + TaskVars.Set("HEAD_SHA", Repository.Head.Tip.Sha); + TaskVars.Set("BRANCH_NAME", Repository.Head.FriendlyName); + TaskVars.Set("BUILD_VERSION", moduleSemVer); + + //Full path to module archive file + TaskVars.Set("FULL_ARCHIVE_FILE_NAME", Path.Combine(WorkingDir.FullName, Config.SourceArchiveName)); + TaskVars.Set("ARCHIVE_FILE_NAME", Config.SourceArchiveName); + TaskVars.Set("ARCHIVE_FILE_FORMAT", Config.SourceArchiveFormat); + + //Remove any previous projects + Projects.Clear(); + + Log.Information("Discovering projects in module {sln}", ModuleName); + + //Discover all projects in for the module + foreach (IProject project in ModuleExplorer.DiscoverProjects()) + { + //Store in collection + Projects.Add(project); + } + + //Load all projects + await Projects.RunAllAsync(p => p.LoadAsync(TaskVars.Clone())); + + Log.Information("Sucessfully loaded {count} projects into module {sln}", Projects.Count, ModuleName); + Log.Information("{modname} CI build SemVer will be {semver}", ModuleName, moduleSemVer); + } + + ///<inheritdoc/> + public virtual async Task DoStepSyncSource() + { + Log.Information("Checking for source code updates in module {mod}", ModuleName); + + //Do a git pull to update our sources + await TaskFile.ExecCommandAsync(this, TaskfileComamnd.Update, true); + + //Set the latest commit sha after an update + TaskVars.Set("HEAD_SHA", Repository.Head.Tip.Sha); + + //Update lastest build number + TaskVars.Set("BUILD_VERSION", this.GetModuleCiVersion(Config.DefaultCiVersion, Config.SemverStyle)); + + //Update module semver after source sync + string moduleSemVer = this.GetModuleCiVersion(Config.DefaultCiVersion, Config.SemverStyle); + TaskVars.Set("BUILD_VERSION", moduleSemVer); + Log.Information("{modname} CI build SemVer will now be {semver}", ModuleName, moduleSemVer); + } + + ///<inheritdoc/> + public virtual async Task<bool> CheckForChangesAsync() + { + //Check source for updates + await Projects.RunAllAsync(p => FileManager.CheckSourceChangedAsync(p, Config, Repository.Head.Tip.Sha)); + + //Check if any project is not up-to-date + return Projects.Any(static p => !p.UpToDate); + } + + ///<inheritdoc/> + public virtual async Task DoStepBuild() + { + //Clean the output dir + FileManager.CleanOutput(); + //Recreate the output dir + FileManager.CreateOutput(); + + //Run taskfile to build + await TaskFile.ExecCommandAsync(this, TaskfileComamnd.Build, true); + + //Run build for all projects + foreach (IProject proj in Projects) + { + //Exec + await TaskFile.ExecCommandAsync(proj, TaskfileComamnd.Build, true); + } + } + + ///<inheritdoc/> + public virtual async Task DoStepPostBuild(bool success) + { + //Run taskfile postbuild, not required to produce a sucessful result + await TaskFile.ExecCommandAsync(this, success ? TaskfileComamnd.PostbuildSuccess : TaskfileComamnd.PostbuildFailure, false); + + //Run postbuild for all projects + foreach (IProject proj in Projects) + { + //Run postbuild for projects + await TaskFile.ExecCommandAsync(proj, success ? TaskfileComamnd.PostbuildSuccess : TaskfileComamnd.PostbuildFailure, false); + } + + //Run postbuild for all projects + await Projects.RunAllAsync(async (p) => + { + //If the operation was a success, commit the sum change + if (success) + { + Log.Verbose("Committing sum change for {sm}", p.ProjectName); + //Commit sum changes now that build has completed successfully + await FileManager.CommitSumChangeAsync(p); + } + }); + } + + ///<inheritdoc/> + public virtual async Task DoStepPublish() + { + //Run taskfile postbuild, not required to produce a sucessful result + await TaskFile.ExecCommandAsync(this, TaskfileComamnd.Publish, true); + + //Run postbuild for all projects + foreach (IProject proj in Projects) + { + //Run postbuild for projects + await TaskFile.ExecCommandAsync(proj, TaskfileComamnd.Publish, true); + } + } + + ///<inheritdoc/> + public virtual async Task DoRunTests(bool failOnError) + { + //Run taskfile to build + await TaskFile.ExecCommandAsync(this, TaskfileComamnd.Test, failOnError); + + //Run build for all projects + foreach (IProject proj in Projects) + { + //Exec + await TaskFile.ExecCommandAsync(proj, TaskfileComamnd.Test, failOnError); + } + } + + ///<inheritdoc/> + public virtual async Task CleanAsync() + { + try + { + //Run taskfile to build + await TaskFile.ExecCommandAsync(this, TaskfileComamnd.Clean, true); + + //Clean all projects + foreach (IProject proj in Projects) + { + //Clean the project output dir + await TaskFile.ExecCommandAsync(proj, TaskfileComamnd.Clean, true); + } + + //Clean the output dir + FileManager.CleanOutput(); + } + catch (BuildStepFailedException) + { + throw; + } + catch (Exception ex) + { + throw new BuildStepFailedException("Failed to remove the module output directory", ex, ModuleName); + } + } + + public override string ToString() => ModuleName; + + + public virtual void Dispose() + { + //Dispose the respository + Repository.Dispose(); + + //empty list + Projects.Clear(); + } + } +}
\ No newline at end of file diff --git a/src/Modules/ModuleFileManager.cs b/src/Modules/ModuleFileManager.cs new file mode 100644 index 0000000..4d6ed69 --- /dev/null +++ b/src/Modules/ModuleFileManager.cs @@ -0,0 +1,92 @@ +using System; +using System.IO; +using System.Threading.Tasks; + +using VNLib.Tools.Build.Executor.Model; +using VNLib.Tools.Build.Executor.Extensions; +using VNLib.Tools.Build.Executor.Constants; + +namespace VNLib.Tools.Build.Executor.Modules +{ + + public sealed class ModuleFileManager(BuildConfig config, IModuleData ModData) : IModuleFileManager + { + private readonly IDirectoryIndex Index = config.Index; + + ///<inheritdoc/> + public string OutputDir => Path.Combine(Index.OutputDir.FullName, ModData.ModuleName); + + ///<inheritdoc/> + public async Task CopyArtifactToOutputAsync(IProject project, FileInfo file) + { + string targetDir = GetProjectTargetDir(project); + + //Project artifacts are versioned by the latest git commit hash + string outputFile = Path.Combine(targetDir, file.Name); + + //Create the target directory if it doesn't exist + Directory.CreateDirectory(targetDir); + + //Copy the file to the output directory + FileInfo output = file.CopyTo(outputFile, true); + + //Compute the file hash of the new output file + await output.ComputeFileHashAsync(config.HashFuncName); + } + + ///<inheritdoc/> + public DirectoryInfo GetArtifactOutputDir(IProject project) + { + string path = GetProjectTargetDir(project); + return new DirectoryInfo(path); + } + + ///<inheritdoc/> + public Task<byte[]?> ReadCheckSumAsync(IProject project) + { + string sumFile = Path.Combine(Index.SumDir.FullName, $"{ModData.ModuleName}-{project.GetSafeProjectName()}.json"); + return File.Exists(sumFile) ? File.ReadAllBytesAsync(sumFile) : Task.FromResult<byte[]?>(null); + } + + ///<inheritdoc/> + public Task WriteChecksumAsync(IProject project, byte[] fileData) + { + //Create sum file inside the sum directory + string sumFile = Path.Combine(Index.SumDir.FullName, $"{ModData.ModuleName}-{project.GetSafeProjectName()}.json"); + return File.WriteAllBytesAsync(sumFile, fileData); + } + + ///<inheritdoc/> + public async Task<FileInfo> WriteFileAsync(ModuleFileType type, byte[] fileData) + { + //Get the file path for the given type + string filePath = type switch + { + //Catalog is written to the version pointed to by the latest git commit hash + ModuleFileType.Catalog => $"{OutputDir}/{GetLatestTagOrSha()}/index.json", + ModuleFileType.GitHistory => $"{OutputDir}/git.json", + ModuleFileType.LatestHash => $"{OutputDir}/@latest", + ModuleFileType.VersionHistory => $"{OutputDir}/versions.json", + //Store project archive + ModuleFileType.Archive => $"{OutputDir}/{GetLatestTagOrSha()}/archive.tgz", + _ => throw new ArgumentOutOfRangeException(nameof(type), type, null), + }; + + await File.WriteAllBytesAsync($"{filePath}", fileData); + //Return new file handle + return new FileInfo(filePath); + } + + private string GetProjectTargetDir(IProject project) + { + //get last tag + return Path.Combine(OutputDir, GetLatestTagOrSha(), project.GetSafeProjectName()); + } + + private string GetLatestTagOrSha() + { + return ModData.Repository.Head.Tip.Sha; + } + + } +}
\ No newline at end of file diff --git a/src/Modules/MsBuildModuleExplorer.cs b/src/Modules/MsBuildModuleExplorer.cs new file mode 100644 index 0000000..fa4592e --- /dev/null +++ b/src/Modules/MsBuildModuleExplorer.cs @@ -0,0 +1,89 @@ +using System; +using System.IO; +using System.Linq; +using System.Collections.Generic; + +using Microsoft.Build.Construction; + +using VNLib.Tools.Build.Executor.Model; +using VNLib.Tools.Build.Executor.Projects; +using VNLib.Tools.Build.Executor.Constants; +using static VNLib.Tools.Build.Executor.Constants.Config; +using VNLib.Tools.Build.Executor.Extensions; + +namespace VNLib.Tools.Build.Executor.Modules +{ + + /* + * Discovers all projects within a dotnet solution file. First it finds the solution file in the + * working directory, then it parses the solution file to find all projects within the solution. + * + * Leaf projects are then discovered by searching for all project files within the working directory + * that are not part of the solution. + */ + + internal sealed class MsBuildModuleExplorer(BuildConfig config, IModuleData Module, DirectoryInfo ModuleDir) : IProjectExplorer + { + ///<inheritdoc/> + public IEnumerable<IProject> DiscoverProjects() + { + LinkedList<IProject> projects = new(); + + //First load the solution file + FileInfo? slnFile = ModuleDir.EnumerateFiles("*.sln", SearchOption.TopDirectoryOnly).FirstOrDefault(); + + if(slnFile is not null) + { + GetProjectsForSoution(slnFile, projects); + } + + //Capture any leaf projects that are not part of the solution + IEnumerable<FileInfo> leafProjects = ModuleDir.EnumerateFiles("package.json", SearchOption.AllDirectories).Distinct(); + + //Capture them + foreach (FileInfo leafProjFile in leafProjects) + { + //Create relative file path + string realtivePath = leafProjFile.FullName.Replace(ModuleDir.FullName, string.Empty).TrimStart(Path.DirectorySeparatorChar); + + //If the leaf project is ignored, skip it + if (Module.Repository.IsFileIgnored(realtivePath)) + { + continue; + } + + //Create the leaf project + LeafProject project = new(config, leafProjFile); + + Log.Verbose("Discovered leaf project in {proj} ", leafProjFile.DirectoryName); + + projects.AddLast(project); + } + + return projects; + } + + private static void GetProjectsForSoution(FileInfo slnFile, LinkedList<IProject> projects) + { + //Parse solution + SolutionFile Solution = SolutionFile.Parse(slnFile.FullName); + + //Loop through all artificats within the solution + foreach (ProjectInSolution proj in Solution.ProjectsInOrder.Where(static p => p.ProjectType == SolutionProjectType.KnownToBeMSBuildFormat)) + { + //Ignore test projects in a solution + if(proj.ProjectName.Contains("test", StringComparison.OrdinalIgnoreCase)) + { + continue; + } + + //Create the new project artifact + DotnetProject project = new(new(proj.AbsolutePath), proj.ProjectName); + + Log.Verbose("Discovered project {proj} ", proj.ProjectName); + + projects.AddLast(project); + } + } + } +}
\ No newline at end of file diff --git a/src/Program.cs b/src/Program.cs new file mode 100644 index 0000000..1a9ad06 --- /dev/null +++ b/src/Program.cs @@ -0,0 +1,38 @@ +using Typin; +using Typin.Console; + +using VNLib.Tools.Build.Executor.Constants; + +using Microsoft.Extensions.DependencyInjection; + + +namespace VNLib.Tools.Build.Executor +{ + + sealed class Program + { + static int Main(string[] argsv) + { + return new CliApplicationBuilder() + .AddCommandsFromThisAssembly() + .UseConsole<SystemConsole>() + .UseTitle("VNBuild Copyright (c) Vaughn Nugent") + .UseStartupMessage("VNBuild Copyright (c) Vaughn Nugent") + .UseVersionText("0.1.0") + .ConfigureServices(services => + { + //Init new pipeline and add to service collection + services + .AddSingleton(Config.Log) + .AddSingleton<BuildPipeline>() + .AddSingleton(new ConfigManager(Semver.SemVersionStyles.Any)); + + }) + .Build() + .RunAsync() + .AsTask() + .GetAwaiter() + .GetResult(); + } + } +}
\ No newline at end of file diff --git a/src/Projects/DotnetProject.cs b/src/Projects/DotnetProject.cs new file mode 100644 index 0000000..5ee1ed0 --- /dev/null +++ b/src/Projects/DotnetProject.cs @@ -0,0 +1,38 @@ + +using System; +using System.IO; +using System.Threading.Tasks; + +using VNLib.Tools.Build.Executor.Model; + +namespace VNLib.Tools.Build.Executor.Projects +{ + + internal sealed class DotnetProject : ModuleProject + { + public override IProjectData ProjectData { get; } + + + public DotnetProject(FileInfo projectFile, string projectName):base(projectFile, projectName) + { + //Create the porject dom + ProjectData = new DotnetProjectDom(); + } + + ///<inheritdoc/> + public override async Task LoadAsync(TaskfileVars vars) + { + //Load project dom + await base.LoadAsync(vars); + + //Set .NET specific vars + TaskVars.Set("TARGET_FRAMEWORK", ProjectData["TargetFramework"] ?? string.Empty); + TaskVars.Set("PROJ_ASM_NAME", ProjectData["AssemblyName"] ?? string.Empty); + } + + public override string ToString() => ProjectName; + + public override void Dispose() + { } + } +}
\ No newline at end of file diff --git a/src/Projects/DotnetProjectDom.cs b/src/Projects/DotnetProjectDom.cs new file mode 100644 index 0000000..9b44747 --- /dev/null +++ b/src/Projects/DotnetProjectDom.cs @@ -0,0 +1,60 @@ + +using System; +using System.IO; +using System.Xml; +using System.Collections.Generic; + +using VNLib.Tools.Build.Executor.Model; + +namespace VNLib.Tools.Build.Executor.Projects +{ + + internal sealed class DotnetProjectDom : IProjectData + { + private readonly XmlDocument _dom; + + public DotnetProjectDom() + { + _dom = new(); + } + + public void Load(Stream stream) + { + _dom.Load(stream); + } + + private XmlNode? project => _dom["Project"]; + + public string? Description => GetProperty("Description"); + public string? Authors => GetProperty("Authors"); + public string? Copyright => GetProperty("Copyright"); + public string? VersionString => GetProperty("Version"); + public string? CompanyName => GetProperty("Company"); + public string? Product => GetProperty("Product"); + public string? RepoUrl => GetProperty("RepositoryUrl"); + + public string? this[string index] => GetProperty(index); + + public string? GetProperty(string name) => GetItemAtPath($"PropertyGroup/{name}"); + + public string? GetItemAtPath(string path) => project!.SelectSingleNode(path)?.InnerText; + + public string[] GetProjectRefs() + { + //Get the item group attr + XmlNodeList? projectRefs = project?.SelectNodes("ItemGroup/ProjectReference"); + + if (projectRefs != null) + { + List<string> refs = new(); + foreach (XmlNode projRef in projectRefs) + { + //Get the project ref via its include attribute + refs.Add(projRef.Attributes["Include"].Value!); + } + return refs.ToArray(); + } + return Array.Empty<string>(); + } + } +}
\ No newline at end of file diff --git a/src/Projects/LeafProject.cs b/src/Projects/LeafProject.cs new file mode 100644 index 0000000..9b155cf --- /dev/null +++ b/src/Projects/LeafProject.cs @@ -0,0 +1,37 @@ +using System.IO; +using System.Threading.Tasks; + +using VNLib.Tools.Build.Executor.Constants; +using VNLib.Tools.Build.Executor.Model; + +namespace VNLib.Tools.Build.Executor.Projects +{ + internal sealed class LeafProject(BuildConfig config, FileInfo projectFile) : ModuleProject(projectFile) + { + ///<inheritdoc/> + public override IProjectData ProjectData { get; } = new NativeProjectDom(); + + ///<inheritdoc/> + protected override FileInfo? PackageInfoFile => new(Path.Combine(WorkingDir.FullName, "package.json")); + + public override async Task LoadAsync(TaskfileVars vars) + { + await base.LoadAsync(vars); + + //Set the project name to the product name if set, otherwise use the working dir name + ProjectName = ProjectData.Product ?? WorkingDir.Name; + + //Get the binary dir from the project file, or use the default + string? binaryDir = ProjectData["output_dir"] ?? ProjectData["output"] ?? config.ProjectBinDir; + + //Overide the project name from the pacakge file if set + TaskVars.Set("PROJECT_NAME", ProjectName); + TaskVars.Set("BINARY_DIR", binaryDir); + } + + public override string ToString() => ProjectName; + + public override void Dispose() + { } + } +}
\ No newline at end of file diff --git a/src/Projects/ModuleProject.cs b/src/Projects/ModuleProject.cs new file mode 100644 index 0000000..4643aa0 --- /dev/null +++ b/src/Projects/ModuleProject.cs @@ -0,0 +1,118 @@ +using System; +using System.IO; +using System.Threading.Tasks; + +using VNLib.Tools.Build.Executor.Model; +using VNLib.Tools.Build.Executor.Extensions; + +namespace VNLib.Tools.Build.Executor.Projects +{ + internal abstract class ModuleProject : IProject + { + ///<inheritdoc/> + public FileInfo ProjectFile { get; } + + ///<inheritdoc/> + public string ProjectName { get; protected set; } + + ///<inheritdoc/> + public abstract IProjectData ProjectData { get; } + + ///<inheritdoc/> + public bool UpToDate { get; set; } + + ///<inheritdoc/> + public DirectoryInfo WorkingDir { get; protected set; } + + ///<inheritdoc/> + public TaskfileVars TaskVars { get; protected set; } + + ///<inheritdoc/> + public string? TaskfileName { get; protected set; } + + /// <summary> + /// Gets the package info file for the project + /// </summary> + protected virtual FileInfo? PackageInfoFile { get; } + + public ModuleProject(FileInfo projectFile, string? projectName = null) + { + ProjectFile = projectFile; + + //Default project name to the file name + ProjectName = projectName ?? Path.GetFileNameWithoutExtension(ProjectFile.Name); + + //Default up-to-date false + UpToDate = false; + + //Default working dir to the project file's directory + WorkingDir = ProjectFile.Directory!; + + TaskVars = null!; + } + + ///<inheritdoc/> + public virtual async Task LoadAsync(TaskfileVars vars) + { + TaskVars = vars; + + await LoadProjectDom(); + + //Set some local environment variables + + //Set local environment variables + TaskVars.Set("PROJECT_NAME", ProjectName); + TaskVars.Set("PROJECT_DIR", WorkingDir.FullName); + TaskVars.Set("IS_PROJECT", bool.TrueString); + + //Store project vars + TaskVars.Set("PROJ_VERSION", ProjectData.VersionString ?? string.Empty); + TaskVars.Set("PROJ_DESCRIPTION", ProjectData.Description ?? string.Empty); + TaskVars.Set("PROJ_AUTHOR", ProjectData.Authors ?? string.Empty); + TaskVars.Set("PROJ_COPYRIGHT", ProjectData.Copyright ?? string.Empty); + TaskVars.Set("PROJ_COMPANY", ProjectData.CompanyName ?? string.Empty); + TaskVars.Set("RPOJ_URL", ProjectData.RepoUrl ?? string.Empty); + + TaskVars.Set("SAFE_PROJ_NAME", this.GetSafeProjectName()); + } + + /// <summary> + /// Loads the project's XML dom from its msbuild project file + /// </summary> + /// <returns>A task that resolves when the dom is built</returns> + public async Task LoadProjectDom() + { + using MemoryStream ms = new(); + + FileInfo dom = ProjectFile; + + if (PackageInfoFile?.Exists == true) + { + dom = PackageInfoFile; + } + + try + { + //Get the project file + await using (FileStream projData = dom.OpenRead()) + { + await projData.CopyToAsync(ms); + } + + //reset stream + ms.Seek(0, SeekOrigin.Begin); + + //Load the project dom + ProjectData.Load(ms); + } + catch (Exception ex) + { + throw new Exception($"Failed to load project dom file {dom.FullName}", ex); + } + } + + public abstract void Dispose(); + + public override string ToString() => ProjectName; + } +}
\ No newline at end of file diff --git a/src/Projects/NativeProjectDom.cs b/src/Projects/NativeProjectDom.cs new file mode 100644 index 0000000..f23a4a5 --- /dev/null +++ b/src/Projects/NativeProjectDom.cs @@ -0,0 +1,57 @@ +using System; +using System.IO; +using System.Text.Json; +using System.Collections.Generic; + +using VNLib.Tools.Build.Executor.Model; + +namespace VNLib.Tools.Build.Executor.Projects +{ + internal sealed class NativeProjectDom : IProjectData + { + private Dictionary<string, string> _properties; + + internal NativeProjectDom() + { + _properties = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase); + } + + public string? this[string index] => _properties.GetValueOrDefault(index); + + public string? Description => this["description"]; + public string? Authors => this["author"]; + public string? Copyright => this["copyright"]; + public string? VersionString => this["version"]; + public string? CompanyName => this["company"]; + public string? Product => this["name"]; + public string? RepoUrl => this["repository"]; + public string? OutputDir => this["output_dir"]; + + public string[] GetProjectRefs() + { + return Array.Empty<string>(); + } + + public void Load(Stream stream) + { + //Read the json file + using JsonDocument doc = JsonDocument.Parse(stream, new JsonDocumentOptions + { + CommentHandling = JsonCommentHandling.Skip, + AllowTrailingCommas = true, + }); + + //Clear old properties + _properties.Clear(); + + //Load new properties that are strings only + foreach (JsonProperty prop in doc.RootElement.EnumerateObject()) + { + if(prop.Value.ValueKind == JsonValueKind.String) + { + _properties[prop.Name] = prop.Value.GetString() ?? string.Empty; + } + } + } + } +}
\ No newline at end of file diff --git a/src/Properties/launchSettings.json b/src/Properties/launchSettings.json new file mode 100644 index 0000000..8701aaa --- /dev/null +++ b/src/Properties/launchSettings.json @@ -0,0 +1,9 @@ +{ + "profiles": { + "VNLib.Tools.Build.Executor": { + "commandName": "Project", + "commandLineArgs": "publish -v -c -x vnlib.browser", + "workingDirectory": "B:\\Progamming\\vnlib.browser" + } + } +}
\ No newline at end of file diff --git a/src/SleetFeedManager.cs b/src/SleetFeedManager.cs new file mode 100644 index 0000000..bc77ff1 --- /dev/null +++ b/src/SleetFeedManager.cs @@ -0,0 +1,53 @@ +using System; +using System.Diagnostics.CodeAnalysis; +using System.IO; +using System.Text.Json; + +using VNLib.Tools.Build.Executor.Model; + +namespace VNLib.Tools.Build.Executor +{ + internal sealed class SleetFeedManager : IFeedManager + { + private readonly string SleetConfigFile; + + ///<inheritdoc/> + public string FeedOutputDir { get; } + + private SleetFeedManager(string indexFilex, string outputDir) + { + //Search for the sleet file in the build dir + SleetConfigFile = indexFilex; + FeedOutputDir = outputDir; + } + + ///<inheritdoc/> + public void AddVariables(TaskfileVars vars) + { + vars.Set("SLEET_DIR", FeedOutputDir); + vars.Set("SLEET_CONFIG_PATH", SleetConfigFile); + } + + /// <summary> + /// Attempts to create a new sleet feed manager from the given directory index + /// if the index contains a sleet feed. Returns null if no sleet feed was found + /// </summary> + /// <returns>The feed manager if found, null otherwise</returns> + /// <exception cref="FileNotFoundException"></exception> + /// <exception cref="ArgumentException"></exception> + [return:NotNullIfNotNull(nameof(feedPath))] + public static IFeedManager? GetSleetFeed(string? feedPath) + { + if(string.IsNullOrWhiteSpace(feedPath)) + { + return null; + } + + //Read the sleet config file + byte[] sleetIndexFile = File.ReadAllBytes(feedPath); + using JsonDocument doc = JsonDocument.Parse(sleetIndexFile); + string rootDir = doc.RootElement.GetProperty("root").GetString() ?? throw new ArgumentException("The sleet output directory was not specified"); + return new SleetFeedManager(feedPath, rootDir); + } + } +}
\ No newline at end of file diff --git a/src/TaskFile.cs b/src/TaskFile.cs new file mode 100644 index 0000000..6eef779 --- /dev/null +++ b/src/TaskFile.cs @@ -0,0 +1,95 @@ +using System; +using System.IO; +using System.Threading.Tasks; +using System.Collections.Generic; + +using VNLib.Tools.Build.Executor.Model; +using VNLib.Tools.Build.Executor.Constants; +using static VNLib.Tools.Build.Executor.Constants.Utils; + +namespace VNLib.Tools.Build.Executor +{ + public enum TaskfileComamnd + { + Clean, + Build, + Upload, + Update, + PostbuildSuccess, + PostbuildFailure, + Publish, + Test, + } + + /// <summary> + /// Represents a controller for the TaskFile build system + /// </summary> + public sealed class TaskFile(string taskFilePath, Func<string> moduleName) + { + /// <summary> + /// Executes the desired Taskfile command with the given user args for + /// the configured manager. + /// </summary> + /// <param name="command">The command to execute</param> + /// <param name="userArgs">Additional user arguments to pass to Task </param> + /// <returns>A task that completes with the status code of the operation</returns> + public async Task ExecCommandAsync(ITaskfileScope scope, TaskfileComamnd command, bool throwIfFailed) + { + //Get working copy of vars + IReadOnlyDictionary<string, string> vars = scope.TaskVars.GetVariables(); + + //Specify taskfile if it is set + List<string> args = []; + if(!string.IsNullOrWhiteSpace(scope.TaskfileName)) + { + //If taskfile is set, we need to make sure it is in the working dir to execute it, otherwise just exit + if(!File.Exists(Path.Combine(scope.WorkingDir.FullName, scope.TaskfileName))) + { + return; + } + + args.Add("-t"); + args.Add(scope.TaskfileName); + } + + //Always add command last + args.Add(GetCommand(command)); + + //Exec task in the module dir + int result = await RunProcessAsync(taskFilePath, scope.WorkingDir.FullName, args.ToArray(), vars); + + if(throwIfFailed) + { + ThrowIfStepFailed(result, command); + } + } + + private static string GetCommand(TaskfileComamnd cmd) + { + return cmd switch + { + TaskfileComamnd.Clean => "clean", + TaskfileComamnd.Build => "build", + TaskfileComamnd.Upload => "upload", + TaskfileComamnd.Update => "update", + TaskfileComamnd.PostbuildSuccess => "postbuild_success", + TaskfileComamnd.PostbuildFailure => "postbuild_failed", + TaskfileComamnd.Publish => "publish", + TaskfileComamnd.Test => "test", + _ => throw new NotImplementedException() + }; + } + + private void ThrowIfStepFailed(int result, TaskfileComamnd cmd) + { + switch (result) + { + case 200: //Named task not found + return; + case 201: + Utils.ThrowIfStepFailed(false, $"Task failed to execute task command {cmd}", moduleName.Invoke()); + return; + } + } + } +}
\ No newline at end of file diff --git a/src/vnbuild.csproj b/src/vnbuild.csproj new file mode 100644 index 0000000..b3e32df --- /dev/null +++ b/src/vnbuild.csproj @@ -0,0 +1,45 @@ +<Project Sdk="Microsoft.NET.Sdk"> + + <PropertyGroup> + <OutputType>Exe</OutputType> + <TargetFramework>net8.0</TargetFramework> + <Nullable>enable</Nullable> + <AssemblyName>vnbuild</AssemblyName> + <RootNamespace>VNLib.Tools.Build.Executor</RootNamespace> + </PropertyGroup> + + <PropertyGroup> + <Authors>Vaughn Nugent</Authors> + <Company>Vaughn Nugent</Company> + <Description>Automatically builds and produces binaries from git-based modules with proper indexing for web based deployments</Description> + <Copyright>Copyright © 2024 Vaughn Nugent</Copyright> + <Product>vnbuild</Product> + <PackageProjectUrl>https://www.vaughnnugent.com/resources/software/modules/vnbuild</PackageProjectUrl> + <RepositoryUrl>https://github.com/VnUgE/vnbuild/</RepositoryUrl> + <PackageReadmeFile>README.md</PackageReadmeFile> + <PackageLicenseFile>LICENSE</PackageLicenseFile> + </PropertyGroup> + + <ItemGroup> + <None Include="..\LICENSE"> + <Pack>True</Pack> + <PackagePath>\</PackagePath> + <CopyToOutputDirectory>Always</CopyToOutputDirectory> + </None> + <None Include="..\README.md"> + <Pack>True</Pack> + <PackagePath>\</PackagePath> + </None> + </ItemGroup> + + <ItemGroup> + <PackageReference Include="Typin" Version="3.1.0" /> + <PackageReference Include="LibGit2Sharp" Version="0.30.0" /> + <PackageReference Include="Microsoft.Build" Version="17.10.0-preview-24081-01" /> + <PackageReference Include="Semver" Version="3.0.0-beta.1" /> + <PackageReference Include="Serilog" Version="4.0.0-dev-02166" /> + <PackageReference Include="Serilog.Sinks.Console" Version="5.1.0-dev-00943" /> + <PackageReference Include="Serilog.Sinks.File" Version="5.0.1-dev-00972" /> + </ItemGroup> + +</Project> diff --git a/vnbuild.build.sln b/vnbuild.build.sln new file mode 100644 index 0000000..bd37a7b --- /dev/null +++ b/vnbuild.build.sln @@ -0,0 +1,35 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio Version 17 +VisualStudioVersion = 17.4.33213.308 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "vnbuild", "src\vnbuild.csproj", "{78010087-AE67-458A-B426-EE78F62CC78C}" +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution Items", "{46AFA632-D1A1-41F5-8C81-7864D6000A2D}" + ProjectSection(SolutionItems) = preProject + .gitattributes = .gitattributes + .gitignore = .gitignore + .onedev-buildspec.yml = .onedev-buildspec.yml + GitVersion.yml = GitVersion.yml + Module.Taskfile.yaml = Module.Taskfile.yaml + Taskfile.yaml = Taskfile.yaml + EndProjectSection +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Release|Any CPU = Release|Any CPU + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {78010087-AE67-458A-B426-EE78F62CC78C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {78010087-AE67-458A-B426-EE78F62CC78C}.Debug|Any CPU.Build.0 = Debug|Any CPU + {78010087-AE67-458A-B426-EE78F62CC78C}.Release|Any CPU.ActiveCfg = Release|Any CPU + {78010087-AE67-458A-B426-EE78F62CC78C}.Release|Any CPU.Build.0 = Release|Any CPU + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection + GlobalSection(ExtensibilityGlobals) = postSolution + SolutionGuid = {00D8043A-78D7-44A7-B85A-6B5827CCB7AF} + EndGlobalSection +EndGlobal |