From 0b69bc760f87efab73ca6efb454b30a3393be269 Mon Sep 17 00:00:00 2001 From: vnugent Date: Sun, 4 Aug 2024 15:36:26 -0400 Subject: consolidate log config --- .../src/Bootstrap/ReleaseWebserver.cs | 7 +- .../src/Config/Model/BenchmarkConfig.cs | 3 +- apps/VNLib.WebServer/src/Config/Model/LogConfig.cs | 52 +++++++++++ .../src/Config/Model/VirtualHostServerConfig.cs | 2 + apps/VNLib.WebServer/src/Entry.cs | 1 - .../src/RuntimeLoading/ServerLogBuilder.cs | 100 +++++++++------------ .../src/VirtualHosts/JsonWebConfigBuilder.cs | 10 ++- apps/VNLib.WebServer/src/sample.config.json | 60 +++++++------ 8 files changed, 138 insertions(+), 97 deletions(-) create mode 100644 apps/VNLib.WebServer/src/Config/Model/LogConfig.cs (limited to 'apps') diff --git a/apps/VNLib.WebServer/src/Bootstrap/ReleaseWebserver.cs b/apps/VNLib.WebServer/src/Bootstrap/ReleaseWebserver.cs index e7834bb..543cbe5 100644 --- a/apps/VNLib.WebServer/src/Bootstrap/ReleaseWebserver.cs +++ b/apps/VNLib.WebServer/src/Bootstrap/ReleaseWebserver.cs @@ -30,7 +30,6 @@ using System.Text.Json; using System.Collections.Generic; using VNLib.Utils.Logging; -using VNLib.Utils.Extensions; using VNLib.Net.Http; using VNLib.Plugins.Runtime; @@ -214,23 +213,19 @@ namespace VNLib.WebServer.Bootstrap /// protected override VirtualHostConfig[] GetAllVirtualHosts() { - JsonElement rootEl = config.GetDocumentRoot(); ILogProvider log = logger.AppLog; LinkedList configs = new(); try { - //execution timeout - TimeSpan execTimeout = rootEl.GetProperty(SESSION_TIMEOUT_PROP_NAME).GetTimeSpan(TimeParseType.Milliseconds); - int index = 0; //Enumerate all virtual host configurations foreach (VirtualHostServerConfig vhConfig in GetVirtualHosts()) { - VirtualHostConfig conf = new JsonWebConfigBuilder(vhConfig, execTimeout, log).GetBaseConfig(); + VirtualHostConfig conf = new JsonWebConfigBuilder(vhConfig, log).GetBaseConfig(); //Configure event hooks conf.EventHooks = new VirtualHostHooks(conf); diff --git a/apps/VNLib.WebServer/src/Config/Model/BenchmarkConfig.cs b/apps/VNLib.WebServer/src/Config/Model/BenchmarkConfig.cs index c569b95..1b0f157 100644 --- a/apps/VNLib.WebServer/src/Config/Model/BenchmarkConfig.cs +++ b/apps/VNLib.WebServer/src/Config/Model/BenchmarkConfig.cs @@ -26,7 +26,8 @@ using System.Text.Json.Serialization; namespace VNLib.WebServer.Config.Model { - internal class BenchmarkConfig + + internal sealed class BenchmarkConfig { [JsonPropertyName("enabled")] diff --git a/apps/VNLib.WebServer/src/Config/Model/LogConfig.cs b/apps/VNLib.WebServer/src/Config/Model/LogConfig.cs new file mode 100644 index 0000000..fafd630 --- /dev/null +++ b/apps/VNLib.WebServer/src/Config/Model/LogConfig.cs @@ -0,0 +1,52 @@ +/* +* Copyright (c) 2024 Vaughn Nugent +* +* Library: VNLib +* Package: VNLib.WebServer +* File: LogConfig.cs +* +* LogConfig.cs is part of VNLib.WebServer which is part of the +* larger VNLib collection of libraries and utilities. +* +* VNLib.WebServer is free software: you can redistribute it and/or modify +* it under the terms of the GNU General Public License as published +* by the Free Software Foundation, either version 2 of the License, +* or (at your option) any later version. +* +* VNLib.WebServer is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +* General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with VNLib.WebServer. If not, see http://www.gnu.org/licenses/. +*/ + +using System.Text.Json.Serialization; + +namespace VNLib.WebServer.Config.Model +{ + internal sealed class LogConfig + { + [JsonPropertyName("enabled")] + public bool Enabled { get; set; } = false; + + [JsonPropertyName("path")] + public string? Path { get; set; } + + [JsonPropertyName("template")] + public string? Template { get; set; } + + [JsonPropertyName("flush_sec")] + public int FlushIntervalSeconds { get; set; } = 10; + + [JsonPropertyName("retained_files")] + public int RetainedFiles { get; set; } = 31; + + [JsonPropertyName("file_size_limit")] + public int FileSizeLimit { get; set; } = 500 * 1000 * 1024; + + [JsonPropertyName("interval")] + public string Interval { get; set; } = "infinite"; + } +} diff --git a/apps/VNLib.WebServer/src/Config/Model/VirtualHostServerConfig.cs b/apps/VNLib.WebServer/src/Config/Model/VirtualHostServerConfig.cs index 2f18cf7..5734bd3 100644 --- a/apps/VNLib.WebServer/src/Config/Model/VirtualHostServerConfig.cs +++ b/apps/VNLib.WebServer/src/Config/Model/VirtualHostServerConfig.cs @@ -92,5 +92,7 @@ namespace VNLib.WebServer.Config.Model [JsonPropertyName("path_filter")] public string? PathFilter { get; set; } + [JsonPropertyName("max_execution_time_ms")] + public int MaxExecutionTimeMs { get; set; } = 20000; } } diff --git a/apps/VNLib.WebServer/src/Entry.cs b/apps/VNLib.WebServer/src/Entry.cs index 731e4f4..1ce660d 100644 --- a/apps/VNLib.WebServer/src/Entry.cs +++ b/apps/VNLib.WebServer/src/Entry.cs @@ -55,7 +55,6 @@ Starting... private static readonly DirectoryInfo EXE_DIR = new(Environment.CurrentDirectory); private const string DEFAULT_CONFIG_PATH = "config.json"; - internal const string SESSION_TIMEOUT_PROP_NAME = "max_execution_time_ms"; internal const string TCP_CONF_PROP_NAME = "tcp"; internal const string LOAD_DEFAULT_HOSTNAME_VALUE = "[system]"; internal const string PLUGINS_CONFIG_PROP_NAME = "plugins"; diff --git a/apps/VNLib.WebServer/src/RuntimeLoading/ServerLogBuilder.cs b/apps/VNLib.WebServer/src/RuntimeLoading/ServerLogBuilder.cs index 4c55258..a10a25e 100644 --- a/apps/VNLib.WebServer/src/RuntimeLoading/ServerLogBuilder.cs +++ b/apps/VNLib.WebServer/src/RuntimeLoading/ServerLogBuilder.cs @@ -24,27 +24,23 @@ using System; using System.IO; -using System.Linq; using System.Text.Json; -using System.Collections.Generic; using Serilog; -using VNLib.Utils.Extensions; +using VNLib.WebServer.Config; +using VNLib.WebServer.Config.Model; namespace VNLib.WebServer.RuntimeLoading { internal sealed class ServerLogBuilder { - public LoggerConfiguration SysLogConfig { get; } - public LoggerConfiguration AppLogConfig { get; } + public LoggerConfiguration SysLogConfig { get; } = new(); + + public LoggerConfiguration AppLogConfig { get; } = new(); + public LoggerConfiguration? DebugConfig { get; } - public ServerLogBuilder() - { - AppLogConfig = new(); - SysLogConfig = new(); - } public ServerLogBuilder BuildForConsole(ProcessArguments args) { @@ -55,8 +51,12 @@ namespace VNLib.WebServer.RuntimeLoading public ServerLogBuilder BuildFromConfig(JsonElement logEl) { - InitSingleLog(logEl, "app_log", "Application", AppLogConfig); - InitSingleLog(logEl, "sys_log", "System", SysLogConfig); + if(logEl.TryGetProperty("logs", out logEl)) + { + InitSingleLog(logEl, "app_log", "Application", AppLogConfig); + InitSingleLog(logEl, "sys_log", "System", SysLogConfig); + } + return this; } @@ -96,58 +96,40 @@ namespace VNLib.WebServer.RuntimeLoading private static void InitSingleLog(JsonElement el, string elPath, string logName, LoggerConfiguration logConfig) { - string? filePath = null; - string? template = null; + if(!el.TryGetProperty(elPath, out el)) + { + return; + } - TimeSpan flushInterval = TimeSpan.FromSeconds(10); - int retainedLogs = 31; - //Default to 500mb log file size - int fileSizeLimit = 500 * 1000 * 1024; - RollingInterval interval = RollingInterval.Infinite; + LogConfig? conf = el.Deserialize(); - //try to get the log config object - if (el.TryGetProperty(elPath, out JsonElement logEl)) + if(conf == null || !conf.Enabled) { - IReadOnlyDictionary conf = logEl.EnumerateObject().ToDictionary(static s => s.Name, static s => s.Value); - - filePath = conf.GetPropString("path"); - template = conf.GetPropString("template"); - - if (conf.TryGetValue("flush_sec", out JsonElement flushEl)) - { - flushInterval = flushEl.GetTimeSpan(TimeParseType.Seconds); - } - - if (conf.TryGetValue("retained_files", out JsonElement retainedEl)) - { - retainedLogs = retainedEl.GetInt32(); - } - - if (conf.TryGetValue("file_size_limit", out JsonElement sizeEl)) - { - fileSizeLimit = sizeEl.GetInt32(); - } - - if (conf.TryGetValue("interval", out JsonElement intervalEl)) - { - interval = Enum.Parse(intervalEl.GetString()!, true); - } - - //Set default objects - filePath ??= Path.Combine(Environment.CurrentDirectory, $"{elPath}.txt"); - template ??= $"{{Timestamp:yyyy-MM-dd HH:mm:ss.fff zzz}} [{{Level:u3}}] {logName} {{Message:lj}}{{NewLine}}{{Exception}}"; - - //Configure the log file writer - logConfig.WriteTo.File(filePath, - buffered: true, - retainedFileCountLimit: retainedLogs, - formatProvider:null, - fileSizeLimitBytes: fileSizeLimit, - rollingInterval: interval, - outputTemplate: template, - flushToDiskInterval: flushInterval); + return; } + //Default path if the user did not set one or set it to null + conf.Path ??= Path.Combine(Environment.CurrentDirectory, $"{elPath}.txt"); + + conf.Template ??= $"{{Timestamp:yyyy-MM-dd HH:mm:ss.fff zzz}} [{{Level:u3}}] {logName} {{Message:lj}}{{NewLine}}{{Exception}}"; + + Validate.EnsureNotNull(conf.Interval, "You must not set a null rolling log interval"); + Validate.EnsureRange(conf.FlushIntervalSeconds, 1, 6000, "flush_sec"); + Validate.EnsureRange(conf.RetainedFiles, 1, 100, "retained_files"); + Validate.EnsureRange(conf.FileSizeLimit, 100, 1000000000, "file_size_limit"); + + //Configure the log file writer + logConfig.WriteTo.File( + path: conf.Path, + buffered: true, + retainedFileCountLimit: conf.RetainedFiles, + formatProvider: null, + fileSizeLimitBytes: conf.FileSizeLimit, + rollingInterval: Enum.Parse(conf.Interval, ignoreCase: true), + outputTemplate: conf.Template, + flushToDiskInterval: TimeSpan.FromSeconds(conf.FlushIntervalSeconds) + ); + //If the log element is not specified in config, do not write log files } } diff --git a/apps/VNLib.WebServer/src/VirtualHosts/JsonWebConfigBuilder.cs b/apps/VNLib.WebServer/src/VirtualHosts/JsonWebConfigBuilder.cs index 820664c..7f3b488 100644 --- a/apps/VNLib.WebServer/src/VirtualHosts/JsonWebConfigBuilder.cs +++ b/apps/VNLib.WebServer/src/VirtualHosts/JsonWebConfigBuilder.cs @@ -40,7 +40,7 @@ using VNLib.WebServer.Config.Model; namespace VNLib.WebServer { - internal sealed partial class JsonWebConfigBuilder(VirtualHostServerConfig VhConfig, TimeSpan execTimeout, ILogProvider logger) + internal sealed partial class JsonWebConfigBuilder(VirtualHostServerConfig VhConfig, ILogProvider logger) : IVirtualHostConfigBuilder { //Use pre-compiled default regex @@ -55,7 +55,7 @@ namespace VNLib.WebServer //File root is required RootDir = new(VhConfig.DirPath!), LogProvider = logger, - ExecutionTimeout = execTimeout, + ExecutionTimeout = GetExecutionTimeout(VhConfig), WhiteList = GetIpWhitelist(VhConfig), DownStreamServers = GetDownStreamServers(VhConfig), ExcludedExtensions = GetExlcudedExtensions(VhConfig), @@ -170,6 +170,12 @@ namespace VNLib.WebServer return (downstreamServers ?? []).ToFrozenSet(); } + private static TimeSpan GetExecutionTimeout(VirtualHostServerConfig conf) + { + //Get the execution timeout + return TimeSpan.FromMilliseconds(conf.MaxExecutionTimeMs); + } + private static FrozenSet? GetIpWhitelist(VirtualHostServerConfig conf) { if(conf.Whitelist is null) diff --git a/apps/VNLib.WebServer/src/sample.config.json b/apps/VNLib.WebServer/src/sample.config.json index a7d268b..10ad783 100644 --- a/apps/VNLib.WebServer/src/sample.config.json +++ b/apps/VNLib.WebServer/src/sample.config.json @@ -18,16 +18,16 @@ "default_version": "HTTP/1.1", //The defaut HTTP version to being requests with (does not support http/2 yet) "multipart_max_buf_size": 20480, //The size of the buffer to use when parsing multipart/form data uploads "multipart_max_size": 80240, //The maxium ammount of data (in bytes) allows for mulitpart/form data file uploads - "max_entity_size": 1024000, //Absolute maximum size (in bytes) of the request entity body (exludes headers) - "keepalive_ms": 1000000, //Keepalive ms for HTTP1.1 keepalive connections + "max_entity_size": 1024000, //Absolute maximum size (in bytes) of the request entity body (exludes headers) "header_buf_size": 8128, //The buffer size to use when parsing headers (also the maxium request header size allowed) "max_request_header_count": 50, //The maxium number of headers allowed in an HTTP request message - "max_connections": 5000, //The maxium number of allowed network connections, before 503s will be issued automatically and connections closed - "recv_timeout_ms": 5000, //time (in ms) to wait for a response from an active connection in recv mode, before dropping it - "send_timeout_ms": 60000, //Time in ms to wait for the client to accept transport data before terminating the connection + "max_connections": 5000, //The maxium number of allowed network connections, before 503s will be issued automatically and connections closed "response_header_buf_size": 16384, //The size (in bytes) of the buffer used to store all response header data "max_uploads_per_request": 10, //Max number of multi-part file uploads allowed per request - + "keepalive_ms": 1000000, //Keepalive ms for HTTP1.1 keepalive connections + "recv_timeout_ms": 5000, //time (in ms) to wait for a response from an active connection in recv mode, before dropping it + "send_timeout_ms": 60000, //Time in ms to wait for the client to accept transport data before terminating the connection + "compression": { "enabled": true, //controls compression globally "assembly": "", //A custom assembly path (ex: 'VNLib.Net.Compression.dll') @@ -36,9 +36,6 @@ } }, - //Maxium ammount of time a request is allowed to be processed (includes loading or waiting for sessions) before operations will be cancelled and a 503 returned - "max_execution_time_ms": 20000, - //Collection of objects to define hosts+interfaces to build server listeners from "virtual_hosts": [ { @@ -48,7 +45,7 @@ //The hostname to listen for, "*" as wildcard, and "[system]" as the default hostname for the current machine. Must be unique "hostnames": [ "*", "localhost" ], - + "trace": false, //Enables connection trace logging for this endpoint "force_port_check": false, //If set, requires the port in the host header to match the transport port @@ -84,7 +81,7 @@ */ "whitelist": [ "127.0.0.1" ], - "blacklist": [ "127.0.0.1" ], //Individual IP addresses to blacklist + "blacklist": [ "127.0.0.1" ], //Individual IP addresses to blacklist //A list of file extensions to deny access to, if a resource is requested and has one of the following extensions, a 404 is returned "deny_extensions": [ ".env", ".htaccess", ".php", ".gitignore" ], @@ -116,7 +113,10 @@ ], //Default http cache time for files - "cache_default_sec": 864000 + "cache_default_sec": 864000, + + //Maxium ammount of time a request is allowed to be processed (includes loading or waiting for sessions) before operations will be cancelled and a 503 returned + "max_execution_time_ms": 20000 } ], @@ -134,22 +134,26 @@ //"config_dir": "" }, - "sys_log": { - //"path": "path/to/syslog/file", - //"template": "serilog template for writing to file", - //"flush_sec": 5, - //"retained_files": 31, - //"file_size_limit": 10485760, - //"interval": "infinite" - }, - - "app_log": { - //"path": "path/to/applog/file", - //"template": "serilog template for writing to file", - //"flush_sec": 5, - //"retained_files": 31, - //"file_size_limit": 10485760, - //"interval": "infinite" + "logs": { + "sys_log": { + "enabled": false, + //"path": "path/to/syslog/file", + //"template": "serilog template for writing to file", + //"flush_sec": 5, + //"retained_files": 31, + //"file_size_limit": 10485760, + //"interval": "infinite" + }, + + "app_log": { + "enabled": false, + //"path": "path/to/applog/file", + //"template": "serilog template for writing to file", + //"flush_sec": 5, + //"retained_files": 31, + //"file_size_limit": 10485760, + //"interval": "infinite" + } }, -- cgit