diff options
Diffstat (limited to 'plugins/VNLib.Plugins.Essentials.Accounts.AppData')
15 files changed, 1142 insertions, 0 deletions
diff --git a/plugins/VNLib.Plugins.Essentials.Accounts.AppData/README.md b/plugins/VNLib.Plugins.Essentials.Accounts.AppData/README.md new file mode 100644 index 0000000..0702226 --- /dev/null +++ b/plugins/VNLib.Plugins.Essentials.Accounts.AppData/README.md @@ -0,0 +1,18 @@ +# VNLib.Plugins.Essentials.Accounts.AppData +*An Essentials plugin that provides endpoints for web-application synchronized storage such as user preferences.* + +> [!WARNING] +> This plugin is still in early development and is not yet ready for production use. + +## Builds +Debug build w/ symbols & xml docs, release builds, NuGet packages, and individually packaged source code are available on my website (link below). + +## Docs and Guides +Documentation, specifications, and setup guides are available on my website. + +[Docs and Articles](https://www.vaughnnugent.com/resources/software/articles?tags=docs,_VNLib.Plugins.Essentials.Accounts.AppData) +[Builds and Source](https://www.vaughnnugent.com/resources/software/modules/Plugins.Essentials) +[Nuget Feeds](https://www.vaughnnugent.com/resources/software/modules) + +## License +Source files in for this project are licensed to you under the GNU Affero General Public License (or any later version). See the LICENSE files for more information.
\ No newline at end of file diff --git a/plugins/VNLib.Plugins.Essentials.Accounts.AppData/build.readme.md b/plugins/VNLib.Plugins.Essentials.Accounts.AppData/build.readme.md new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/plugins/VNLib.Plugins.Essentials.Accounts.AppData/build.readme.md diff --git a/plugins/VNLib.Plugins.Essentials.Accounts.AppData/src/AppDataEntry.cs b/plugins/VNLib.Plugins.Essentials.Accounts.AppData/src/AppDataEntry.cs new file mode 100644 index 0000000..d6d936f --- /dev/null +++ b/plugins/VNLib.Plugins.Essentials.Accounts.AppData/src/AppDataEntry.cs @@ -0,0 +1,54 @@ +/* +* Copyright (c) 2024 Vaughn Nugent +* +* Library: VNLib +* Package: VNLib.Plugins.Essentials.Accounts.AppData +* File: AppDataEntry.cs +* +* AppDataEntry.cs is part of VNLib.Plugins.Essentials.Accounts.AppData which +* is part of the larger VNLib collection of libraries and utilities. +* +* VNLib.Plugins.Essentials.Accounts is free software: you can redistribute it and/or modify +* it under the terms of the GNU Affero General Public License as +* published by the Free Software Foundation, either version 3 of the +* License, or (at your option) any later version. +* +* VNLib.Plugins.Essentials.Accounts is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Affero General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see https://www.gnu.org/licenses/. +*/ + +using VNLib.Utils.Logging; +using VNLib.Plugins.Extensions.Loading.Routing; + +using VNLib.Plugins.Essentials.Accounts.AppData.Endpoints; + +namespace VNLib.Plugins.Essentials.Accounts.AppData +{ + public sealed class AppDataEntry : PluginBase + { + ///<inheritdoc/> + public override string PluginName => "Essentials.AppData"; + + ///<inheritdoc/> + protected override void OnLoad() + { + this.Route<WebEndpoint>(); + Log.Information("Plugin loaded"); + } + + ///<inheritdoc/> + protected override void OnUnLoad() + { + Log.Information("Plugin unloaded"); + } + + ///<inheritdoc/> + protected override void ProcessHostCommand(string cmd) + { } + } +} diff --git a/plugins/VNLib.Plugins.Essentials.Accounts.AppData/src/CacheStore.cs b/plugins/VNLib.Plugins.Essentials.Accounts.AppData/src/CacheStore.cs new file mode 100644 index 0000000..95c1b5a --- /dev/null +++ b/plugins/VNLib.Plugins.Essentials.Accounts.AppData/src/CacheStore.cs @@ -0,0 +1,249 @@ +/* +* Copyright (c) 2024 Vaughn Nugent +* +* Library: VNLib +* Package: VNLib.Plugins.Essentials.Accounts.AppData +* File: CacheStore.cs +* +* CacheStore.cs is part of VNLib.Plugins.Essentials.Accounts.AppData which +* is part of the larger VNLib collection of libraries and utilities. +* +* VNLib.Plugins.Essentials.Accounts is free software: you can redistribute it and/or modify +* it under the terms of the GNU Affero General Public License as +* published by the Free Software Foundation, either version 3 of the +* License, or (at your option) any later version. +* +* VNLib.Plugins.Essentials.Accounts is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Affero General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see https://www.gnu.org/licenses/. +*/ + +using System; +using System.Buffers; +using System.Threading; +using System.Threading.Tasks; + +using MemoryPack; + +using VNLib.Utils.Extensions; +using VNLib.Utils.Logging; +using VNLib.Hashing.Checksums; +using VNLib.Data.Caching; +using VNLib.Plugins.Extensions.Loading; +using VNLib.Plugins.Extensions.VNCache; +using VNLib.Plugins.Extensions.VNCache.DataModel; +using VNLib.Plugins.Essentials.Accounts.AppData.Stores; +using VNLib.Plugins.Essentials.Accounts.AppData.Model; + +namespace VNLib.Plugins.Essentials.Accounts.AppData +{ + + [ConfigurationName("record_cache")] + internal sealed class CacheStore : IAppDataStore + { + const string LogScope = "Record Cache"; + + private readonly IEntityCache<RecordDataCacheEntry> _cache; + private readonly PersistentStorageManager _backingStore; + private readonly ILogProvider _logger; + private readonly bool AlwaysObserverCacheUpdate; + private readonly TimeSpan CacheTTL; + + + public CacheStore(PluginBase plugin, IConfigScope config) + { + string cachePrefix = config.GetRequiredProperty("prefix", p => p.GetString()!); + CacheTTL = config.GetRequiredProperty("ttl", p => p.GetTimeSpan(TimeParseType.Seconds))!; + AlwaysObserverCacheUpdate = config.GetRequiredProperty("force_write_through", p => p.GetBoolean())!; + _logger = plugin.Log.CreateScope(LogScope); + + //Load persistent storage manager + _backingStore = plugin.GetOrCreateSingleton<PersistentStorageManager>(); + + //Use memory pack for serialization + MpSerializer serializer = new(); + + /* + * Initialize entity cache from the default global cache provider, + * then create a prefixed cache for the app data records. + * + * The app should make sure that the cache provider is available + * otherwise do not load this component. + */ + _cache = plugin.GetDefaultGlobalCache() + ?.GetPrefixedCache(cachePrefix) + ?.CreateEntityCache<RecordDataCacheEntry>(serializer, serializer) + ?? throw new InvalidOperationException("No cache provider is available"); + + _logger.Verbose("Cache and backing store initialized"); + } + + ///<inheritdoc/> + public Task DeleteRecordAsync(string userId, string recordKey, CancellationToken cancellation) + { + /* + * Deleting entires does not matter if they existed previously or not. Just + * that the opeation executed successfully. + * + * Parallelize the delete operation to the cache and the backing store + */ + Task fromCache = _cache.RemoveAsync(GetCacheKey(userId, recordKey), cancellation); + Task fromDb = _backingStore.DeleteRecordAsync(userId, recordKey, cancellation); + + return Task.WhenAll(fromCache, fromDb); + } + + ///<inheritdoc/> + public async Task<UserRecordData?> GetRecordAsync(string userId, string recordKey, RecordOpFlags flags, CancellationToken cancellation) + { + bool useCache = (flags & RecordOpFlags.NoCache) == 0; + + //See if caller wants to bypass cache + if (useCache) + { + string cacheKey = GetCacheKey(userId, recordKey); + + //try fetching from cache + RecordDataCacheEntry? cached = await _cache.GetAsync(cacheKey, cancellation); + + //if cache is valid, return it + if (cached != null && !IsCacheExpired(cached)) + { + return new(userId, cached.RecordData, cached.UnixTimestamp, cached.Checksum); + } + } + + //fetch from db + UserRecordData? stored = await _backingStore.GetRecordAsync(userId, recordKey, flags, cancellation); + + //If the record is valid and cache is enabled, update the record in cache + if (useCache && stored is not null) + { + //If no checksum is present, calculate it before storing in cache + if (!stored.Checksum.HasValue) + { + ulong checksum = FNV1a.Compute64(stored.Data); + stored = stored with { Checksum = checksum }; + } + + //update cached version + Task update = DeferCacheUpdate( + userId, + recordKey, + stored.Data, + stored.LastModifed, + stored.Checksum.Value + ); + + if (AlwaysObserverCacheUpdate || (flags & RecordOpFlags.WriteThrough) != 0) + { + //Wait for cache update to complete + await update.ConfigureAwait(false); + } + else + { + //Defer the cache update and continue + WatchDeferredCacheUpdate(update); + } + } + + return stored; + } + + ///<inheritdoc/> + public Task SetRecordAsync(string userId, string recordKey, byte[] data, ulong checksum, RecordOpFlags flags, CancellationToken cancellation) + { + + //Always push update to db + Task db = _backingStore.SetRecordAsync(userId, recordKey, data, checksum, flags, cancellation); + + //Optionally push update to cache + Task cache = Task.CompletedTask; + + if ((flags & RecordOpFlags.NoCache) == 0) + { + long time = DateTimeOffset.Now.ToUnixTimeSeconds(); + + //Push update to cache + cache = DeferCacheUpdate(userId, recordKey, data, time, checksum); + } + + /* + * If writethough is not set, updates will always be deferred + * and this call will return immediately. + * + * We still need to observe the task incase an error occurs + */ + Task all = Task.WhenAll(db, cache); + + if (AlwaysObserverCacheUpdate || (flags & RecordOpFlags.WriteThrough) != 0) + { + return all; + } + else + { + WatchDeferredCacheUpdate(all); + return Task.CompletedTask; + } + } + + private string GetCacheKey(string userId, string recordKey) => $"{userId}:{recordKey}"; + + private bool IsCacheExpired(RecordDataCacheEntry entry) + { + return DateTimeOffset.FromUnixTimeSeconds(entry.UnixTimestamp).Add(CacheTTL) < DateTimeOffset.Now; + } + + private Task DeferCacheUpdate(string userId, string recordKey, byte[] data, long time, ulong checksum) + { + string cacheKey = GetCacheKey(userId, recordKey); + + RecordDataCacheEntry entry = new() + { + Checksum = checksum, + RecordData = data, + UnixTimestamp = time + }; + + return _cache.UpsertAsync(cacheKey, entry); + } + + private async void WatchDeferredCacheUpdate(Task update) + { + try + { + await update.ConfigureAwait(false); + } + catch (Exception e) + { + if (_logger.IsEnabled(LogLevel.Debug)) + { + _logger.Warn(e, "Failed to update cached User AppData record"); + } + else + { + _logger.Warn("Failed to update cached AppData record"); + } + } + } + + + private sealed class MpSerializer : ICacheObjectDeserializer, ICacheObjectSerializer + { + + public T? Deserialize<T>(ReadOnlySpan<byte> objectData) + { + return MemoryPackSerializer.Deserialize<T>(objectData); + } + + public void Serialize<T>(T obj, IBufferWriter<byte> finiteWriter) + { + MemoryPackSerializer.Serialize(finiteWriter, obj); + } + } + } +} diff --git a/plugins/VNLib.Plugins.Essentials.Accounts.AppData/src/Endpoints/WebEndpoint.cs b/plugins/VNLib.Plugins.Essentials.Accounts.AppData/src/Endpoints/WebEndpoint.cs new file mode 100644 index 0000000..9c8f501 --- /dev/null +++ b/plugins/VNLib.Plugins.Essentials.Accounts.AppData/src/Endpoints/WebEndpoint.cs @@ -0,0 +1,203 @@ +/* +* Copyright (c) 2024 Vaughn Nugent +* +* Library: VNLib +* Package: VNLib.Plugins.Essentials.Accounts.AppData +* File: WebEndpoint.cs +* +* WebEndpoint.cs is part of VNLib.Plugins.Essentials.Accounts.AppData which +* is part of the larger VNLib collection of libraries and utilities. +* +* VNLib.Plugins.Essentials.Accounts is free software: you can redistribute it and/or modify +* it under the terms of the GNU Affero General Public License as +* published by the Free Software Foundation, either version 3 of the +* License, or (at your option) any later version. +* +* VNLib.Plugins.Essentials.Accounts is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Affero General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see https://www.gnu.org/licenses/. +*/ + +using System.Net; +using System.Linq; +using System.Collections.Generic; +using System.Threading.Tasks; + +using VNLib.Net.Http; +using VNLib.Utils.Logging; +using VNLib.Hashing.Checksums; +using VNLib.Plugins.Essentials.Endpoints; +using VNLib.Plugins.Essentials.Extensions; +using VNLib.Plugins.Extensions.VNCache; +using VNLib.Plugins.Extensions.Loading; +using VNLib.Plugins.Extensions.Validation; + +using VNLib.Plugins.Essentials.Accounts.AppData.Model; +using VNLib.Plugins.Essentials.Accounts.AppData.Stores; + +namespace VNLib.Plugins.Essentials.Accounts.AppData.Endpoints +{ + [ConfigurationName("web_endpoint")] + internal sealed class WebEndpoint : ProtectedWebEndpoint + { + const int DefaultMaxDataSize = 8 * 1024; + + private readonly IAppDataStore _store; + private readonly int MaxDataSize; + private readonly string[] AllowedScopes; + + public WebEndpoint(PluginBase plugin, IConfigScope config) + { + string path = config.GetRequiredProperty("path", p => p.GetString())!; + InitPathAndLog(path, plugin.Log.CreateScope("Endpoint")); + + MaxDataSize = config.GetValueOrDefault("max_data_size", p => p.GetInt32(), DefaultMaxDataSize); + AllowedScopes = config.GetRequiredProperty("allowed_scopes", p => p.EnumerateArray().Select(p => p.GetString()!)).ToArray(); + + bool useCache = false; + + //Cache loading is optional + if (plugin.HasConfigForType<CacheStore>()) + { + //See if caching is enabled + IConfigScope cacheConfig = plugin.GetConfigForType<CacheStore>(); + useCache = cacheConfig.GetValueOrDefault("enabled", e => e.GetBoolean(), false); + + if (useCache && plugin.GetDefaultGlobalCache() is null) + { + plugin.Log.Error("Cache was enabled but no caching library was loaded. Continuing without cache"); + useCache = false; + } + } + + _store = LoadStore(plugin, useCache); + } + + private static IAppDataStore LoadStore(PluginBase plugin, bool withCache) + { + return withCache + ? plugin.GetOrCreateSingleton<CacheStore>() + : plugin.GetOrCreateSingleton<PersistentStorageManager>(); + } + + protected async override ValueTask<VfReturnType> GetAsync(HttpEntity entity) + { + WebMessage webm = new(); + + string? scopeId = entity.QueryArgs.GetValueOrDefault("scope"); + bool noCache = entity.QueryArgs.ContainsKey("no_cache"); + + if (webm.Assert(scopeId != null, "Missing scope")) + { + return VirtualClose(entity, webm, HttpStatusCode.BadRequest); + } + + if (webm.Assert(AllowedScopes.Contains(scopeId), "Invalid scope")) + { + return VirtualClose(entity, webm, HttpStatusCode.BadRequest); + } + + //If the connection has the no-cache header set, also bypass the cache + noCache |= entity.Server.NoCache(); + + //optionally bypass cache if the user requests it + RecordOpFlags flags = noCache ? RecordOpFlags.NoCache : RecordOpFlags.None; + + UserRecordData? record = await _store.GetRecordAsync(entity.Session.UserID, scopeId, flags, entity.EventCancellation); + + if (record is null) + { + return VirtualClose(entity, webm, HttpStatusCode.NotFound); + } + + //return the raw data with the checksum header + entity.SetRecordResponse(record, HttpStatusCode.OK); + return VfReturnType.VirtualSkip; + } + + protected override async ValueTask<VfReturnType> PutAsync(HttpEntity entity) + { + WebMessage webm = new(); + string? scopeId = entity.QueryArgs.GetValueOrDefault("scope"); + bool flush = entity.QueryArgs.ContainsKey("flush"); + + if (webm.Assert(entity.Files.Count == 1, "Invalid file count")) + { + return VirtualClose(entity, webm, HttpStatusCode.BadRequest); + } + + if (webm.Assert(scopeId != null, "Missing scope")) + { + return VirtualClose(entity, webm, HttpStatusCode.BadRequest); + } + + if (webm.Assert(AllowedScopes.Contains(scopeId), "Invalid scope")) + { + return VirtualClose(entity, webm, HttpStatusCode.BadRequest); + } + + FileUpload data = entity.Files[0]; + + if (webm.Assert(data.Length <= MaxDataSize, "Data too large")) + { + return VirtualClose(entity, webm, HttpStatusCode.RequestEntityTooLarge); + } + + byte[] recordData = new byte[data.Length]; + int read = await data.FileData.ReadAsync(recordData, entity.EventCancellation); + + if (webm.Assert(read == recordData.Length, "Failed to read data")) + { + return VirtualClose(entity, webm, HttpStatusCode.InternalServerError); + } + + //Compute checksum on sent data and compare to the header if it exists + ulong checksum = FNV1a.Compute64(recordData); + ulong? userChecksum = entity.Server.GetUserDataChecksum(); + + if (userChecksum.HasValue) + { + //compare the checksums + if (webm.Assert(checksum == userChecksum.Value, "Checksum mismatch")) + { + return VirtualClose(entity, webm, HttpStatusCode.UnprocessableEntity); + } + } + + /* + * If the user specifies the flush flag, the call will wait until the entire record + * is published to the persistent store before returning. Typically if a caching layer is + * used, the record will be written to the cache and the call will return immediately. + */ + RecordOpFlags flags = flush ? RecordOpFlags.WriteThrough : RecordOpFlags.None; + + //Write the record to the store + await _store.SetRecordAsync(entity.Session.UserID, scopeId, recordData, checksum, flags, entity.EventCancellation); + return VirtualClose(entity, HttpStatusCode.Accepted); + } + + protected override async ValueTask<VfReturnType> DeleteAsync(HttpEntity entity) + { + WebMessage webm = new(); + string? scopeId = entity.QueryArgs.GetValueOrDefault("scope"); + + if (webm.Assert(scopeId != null, "Missing scope")) + { + return VirtualClose(entity, webm, HttpStatusCode.BadRequest); + } + + if (webm.Assert(AllowedScopes.Contains(scopeId), "Invalid scope")) + { + return VirtualClose(entity, webm, HttpStatusCode.BadRequest); + } + + //Write the record to the store + await _store.DeleteRecordAsync(entity.Session.UserID, scopeId, entity.EventCancellation); + return VirtualClose(entity, HttpStatusCode.Accepted); + } + } +} diff --git a/plugins/VNLib.Plugins.Essentials.Accounts.AppData/src/Model/HttpExtensions.cs b/plugins/VNLib.Plugins.Essentials.Accounts.AppData/src/Model/HttpExtensions.cs new file mode 100644 index 0000000..9628b79 --- /dev/null +++ b/plugins/VNLib.Plugins.Essentials.Accounts.AppData/src/Model/HttpExtensions.cs @@ -0,0 +1,75 @@ +/* +* Copyright (c) 2024 Vaughn Nugent +* +* Library: VNLib +* Package: VNLib.Plugins.Essentials.Accounts.AppData +* File: HttpExtensions.cs +* +* HttpExtensions.cs is part of VNLib.Plugins.Essentials.Accounts.AppData which +* is part of the larger VNLib collection of libraries and utilities. +* +* VNLib.Plugins.Essentials.Accounts is free software: you can redistribute it and/or modify +* it under the terms of the GNU Affero General Public License as +* published by the Free Software Foundation, either version 3 of the +* License, or (at your option) any later version. +* +* VNLib.Plugins.Essentials.Accounts is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Affero General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see https://www.gnu.org/licenses/. +*/ + +using System; +using System.Net; + +using VNLib.Net.Http; + +namespace VNLib.Plugins.Essentials.Accounts.AppData.Model +{ + internal static class HttpExtensions + { + const string ChecksumHeader = "X-Data-Checksum"; + + public static void SetRecordResponse(this HttpEntity entity, UserRecordData record, HttpStatusCode code) + { + //Set checksum header + entity.Server.Headers.Append(ChecksumHeader, $"{record.Checksum}"); + + //Set the response to a new memory reader with the record data + entity.CloseResponse( + code, + ContentType.Binary, + new BinDataRecordReader(record.Data) + ); + } + + public static ulong? GetUserDataChecksum(this IConnectionInfo server) + { + string? checksumStr = server.Headers[ChecksumHeader]; + return string.IsNullOrWhiteSpace(checksumStr) && ulong.TryParse(checksumStr, out ulong checksum) ? checksum : null; + } + + sealed class BinDataRecordReader(byte[] recordData) : IMemoryResponseReader + { + private int _read; + + ///<inheritdoc/> + public int Remaining => recordData.Length - _read; + + ///<inheritdoc/> + public void Advance(int written) => _read += written; + + ///<inheritdoc/> + public void Close() + { + //No-op + } + + ///<inheritdoc/> + public ReadOnlyMemory<byte> GetMemory() => recordData.AsMemory(_read); + } + } +} diff --git a/plugins/VNLib.Plugins.Essentials.Accounts.AppData/src/Model/IAppDataStore.cs b/plugins/VNLib.Plugins.Essentials.Accounts.AppData/src/Model/IAppDataStore.cs new file mode 100644 index 0000000..5b38d21 --- /dev/null +++ b/plugins/VNLib.Plugins.Essentials.Accounts.AppData/src/Model/IAppDataStore.cs @@ -0,0 +1,38 @@ +/* +* Copyright (c) 2024 Vaughn Nugent +* +* Library: VNLib +* Package: VNLib.Plugins.Essentials.Accounts.AppData +* File: IAppDataStore.cs +* +* IAppDataStore.cs is part of VNLib.Plugins.Essentials.Accounts.AppData which +* is part of the larger VNLib collection of libraries and utilities. +* +* VNLib.Plugins.Essentials.Accounts is free software: you can redistribute it and/or modify +* it under the terms of the GNU Affero General Public License as +* published by the Free Software Foundation, either version 3 of the +* License, or (at your option) any later version. +* +* VNLib.Plugins.Essentials.Accounts is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Affero General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see https://www.gnu.org/licenses/. +*/ + +using System.Threading; +using System.Threading.Tasks; + +namespace VNLib.Plugins.Essentials.Accounts.AppData.Model +{ + internal interface IAppDataStore + { + Task<UserRecordData?> GetRecordAsync(string userId, string recordKey, RecordOpFlags flags, CancellationToken cancellation); + + Task SetRecordAsync(string userId, string recordKey, byte[] data, ulong checksum, RecordOpFlags flags, CancellationToken cancellation); + + Task DeleteRecordAsync(string userId, string recordKey, CancellationToken cancellation); + } +} diff --git a/plugins/VNLib.Plugins.Essentials.Accounts.AppData/src/Model/RecordDataCacheEntry.cs b/plugins/VNLib.Plugins.Essentials.Accounts.AppData/src/Model/RecordDataCacheEntry.cs new file mode 100644 index 0000000..9c0767d --- /dev/null +++ b/plugins/VNLib.Plugins.Essentials.Accounts.AppData/src/Model/RecordDataCacheEntry.cs @@ -0,0 +1,38 @@ +/* +* Copyright (c) 2024 Vaughn Nugent +* +* Library: VNLib +* Package: VNLib.Plugins.Essentials.Accounts.AppData +* File: RecordDataCacheEntry.cs +* +* RecordDataCacheEntry.cs is part of VNLib.Plugins.Essentials.Accounts.AppData which +* is part of the larger VNLib collection of libraries and utilities. +* +* VNLib.Plugins.Essentials.Accounts is free software: you can redistribute it and/or modify +* it under the terms of the GNU Affero General Public License as +* published by the Free Software Foundation, either version 3 of the +* License, or (at your option) any later version. +* +* VNLib.Plugins.Essentials.Accounts is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Affero General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see https://www.gnu.org/licenses/. +*/ + +using MemoryPack; + +namespace VNLib.Plugins.Essentials.Accounts.AppData.Model +{ + [MemoryPackable] + internal partial class RecordDataCacheEntry + { + public byte[] RecordData { get; set; } + + public ulong? Checksum { get; set; } + + public long UnixTimestamp { get; set; } + } +} diff --git a/plugins/VNLib.Plugins.Essentials.Accounts.AppData/src/Model/RecordOpFlags.cs b/plugins/VNLib.Plugins.Essentials.Accounts.AppData/src/Model/RecordOpFlags.cs new file mode 100644 index 0000000..31d9840 --- /dev/null +++ b/plugins/VNLib.Plugins.Essentials.Accounts.AppData/src/Model/RecordOpFlags.cs @@ -0,0 +1,35 @@ +/* +* Copyright (c) 2024 Vaughn Nugent +* +* Library: VNLib +* Package: VNLib.Plugins.Essentials.Accounts.AppData +* File: RecordOpFlags.cs +* +* RecordOpFlags.cs is part of VNLib.Plugins.Essentials.Accounts.AppData which +* is part of the larger VNLib collection of libraries and utilities. +* +* VNLib.Plugins.Essentials.Accounts is free software: you can redistribute it and/or modify +* it under the terms of the GNU Affero General Public License as +* published by the Free Software Foundation, either version 3 of the +* License, or (at your option) any later version. +* +* VNLib.Plugins.Essentials.Accounts is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Affero General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see https://www.gnu.org/licenses/. +*/ + + +namespace VNLib.Plugins.Essentials.Accounts.AppData.Model +{ + internal enum RecordOpFlags + { + None = 0, + IgnoreChecksum = 1, + WriteThrough = 2, + NoCache = 4, + } +} diff --git a/plugins/VNLib.Plugins.Essentials.Accounts.AppData/src/Model/UserRecordData.cs b/plugins/VNLib.Plugins.Essentials.Accounts.AppData/src/Model/UserRecordData.cs new file mode 100644 index 0000000..d3770c6 --- /dev/null +++ b/plugins/VNLib.Plugins.Essentials.Accounts.AppData/src/Model/UserRecordData.cs @@ -0,0 +1,28 @@ +/* +* Copyright (c) 2024 Vaughn Nugent +* +* Library: VNLib +* Package: VNLib.Plugins.Essentials.Accounts.AppData +* File: UserRecordData.cs +* +* UserRecordData.cs is part of VNLib.Plugins.Essentials.Accounts.AppData which +* is part of the larger VNLib collection of libraries and utilities. +* +* VNLib.Plugins.Essentials.Accounts is free software: you can redistribute it and/or modify +* it under the terms of the GNU Affero General Public License as +* published by the Free Software Foundation, either version 3 of the +* License, or (at your option) any later version. +* +* VNLib.Plugins.Essentials.Accounts is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Affero General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see https://www.gnu.org/licenses/. +*/ + +namespace VNLib.Plugins.Essentials.Accounts.AppData.Model +{ + internal record class UserRecordData(string UserId, byte[] Data, long LastModifed, ulong? Checksum); +} diff --git a/plugins/VNLib.Plugins.Essentials.Accounts.AppData/src/Stores/PersistentStorageManager.cs b/plugins/VNLib.Plugins.Essentials.Accounts.AppData/src/Stores/PersistentStorageManager.cs new file mode 100644 index 0000000..99a3286 --- /dev/null +++ b/plugins/VNLib.Plugins.Essentials.Accounts.AppData/src/Stores/PersistentStorageManager.cs @@ -0,0 +1,75 @@ +/* +* Copyright (c) 2024 Vaughn Nugent +* +* Library: VNLib +* Package: VNLib.Plugins.Essentials.Accounts.AppData +* File: PersistentStorageManager.cs +* +* PersistentStorageManager.cs is part of VNLib.Plugins.Essentials.Accounts.AppData which +* is part of the larger VNLib collection of libraries and utilities. +* +* VNLib.Plugins.Essentials.Accounts is free software: you can redistribute it and/or modify +* it under the terms of the GNU Affero General Public License as +* published by the Free Software Foundation, either version 3 of the +* License, or (at your option) any later version. +* +* VNLib.Plugins.Essentials.Accounts is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Affero General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see https://www.gnu.org/licenses/. +*/ + +using System; +using System.Threading; +using System.Threading.Tasks; + +using VNLib.Utils.Logging; +using VNLib.Plugins.Extensions.Loading; + +using VNLib.Plugins.Essentials.Accounts.AppData.Model; +using VNLib.Plugins.Essentials.Accounts.AppData.Stores.Sql; + +namespace VNLib.Plugins.Essentials.Accounts.AppData.Stores +{ + [ConfigurationName("storage")] + internal sealed class PersistentStorageManager : IAppDataStore + { + private readonly IAppDataStore _backingStore; + + public PersistentStorageManager(PluginBase plugin, IConfigScope config) + { + string storeType = config.GetRequiredProperty("type", p => p.GetString()!).ToLower(null); + + switch (storeType) + { + case "sql": + _backingStore = plugin.GetOrCreateSingleton<SqlBackingStore>(); + plugin.Log.Information("Using SQL based backing store"); + break; + default: + throw new NotSupportedException($"Storage type {storeType} is not supported"); + } + } + + ///<inheritdoc/> + public Task DeleteRecordAsync(string userId, string recordKey, CancellationToken cancellation) + { + return _backingStore.DeleteRecordAsync(userId, recordKey, cancellation); + } + + ///<inheritdoc/> + public Task<UserRecordData?> GetRecordAsync(string userId, string recordKey, RecordOpFlags flags, CancellationToken cancellation) + { + return _backingStore.GetRecordAsync(userId, recordKey, flags, cancellation); + } + + ///<inheritdoc/> + public Task SetRecordAsync(string userId, string recordKey, byte[] data, ulong checksum, RecordOpFlags flags, CancellationToken cancellation) + { + return _backingStore.SetRecordAsync(userId, recordKey, data, checksum, flags, cancellation); + } + } +} diff --git a/plugins/VNLib.Plugins.Essentials.Accounts.AppData/src/Stores/Sql/DataRecord.cs b/plugins/VNLib.Plugins.Essentials.Accounts.AppData/src/Stores/Sql/DataRecord.cs new file mode 100644 index 0000000..f3be974 --- /dev/null +++ b/plugins/VNLib.Plugins.Essentials.Accounts.AppData/src/Stores/Sql/DataRecord.cs @@ -0,0 +1,59 @@ +/* +* Copyright (c) 2024 Vaughn Nugent +* +* Library: VNLib +* Package: VNLib.Plugins.Essentials.Accounts.AppData +* File: DataRecord.cs +* +* DataRecord.cs is part of VNLib.Plugins.Essentials.Accounts.AppData which +* is part of the larger VNLib collection of libraries and utilities. +* +* VNLib.Plugins.Essentials.Accounts is free software: you can redistribute it and/or modify +* it under the terms of the GNU Affero General Public License as +* published by the Free Software Foundation, either version 3 of the +* License, or (at your option) any later version. +* +* VNLib.Plugins.Essentials.Accounts is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Affero General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see https://www.gnu.org/licenses/. +*/ + +using System; +using System.ComponentModel.DataAnnotations; + +using VNLib.Plugins.Extensions.Data.Abstractions; +using VNLib.Plugins.Extensions.Data; + +namespace VNLib.Plugins.Essentials.Accounts.AppData.Stores.Sql +{ + +#nullable disable + internal sealed class DataRecord : DbModelBase, IUserEntity + { + [Key] + [MaxLength(64)] + public override string Id { get; set; } + + [MaxLength(64)] + public string RecordKey { get; set; } + + [MaxLength(64)] + public string UserId { get; set; } + + public override DateTime Created { get; set; } + + public override DateTime LastModified { get; set; } + + [MaxLength(int.MaxValue)] //Should defailt to MAX it set to very large number + public byte[] Data { get; set; } + + /// <summary> + /// The FNV-1a checksum of the data + /// </summary> + public long Checksum { get; set; } + } +} diff --git a/plugins/VNLib.Plugins.Essentials.Accounts.AppData/src/Stores/Sql/SqlBackingStore.cs b/plugins/VNLib.Plugins.Essentials.Accounts.AppData/src/Stores/Sql/SqlBackingStore.cs new file mode 100644 index 0000000..f67c652 --- /dev/null +++ b/plugins/VNLib.Plugins.Essentials.Accounts.AppData/src/Stores/Sql/SqlBackingStore.cs @@ -0,0 +1,144 @@ +/* +* Copyright (c) 2024 Vaughn Nugent +* +* Library: VNLib +* Package: VNLib.Plugins.Essentials.Accounts.AppData +* File: SqlBackingStore.cs +* +* SqlBackingStore.cs is part of VNLib.Plugins.Essentials.Accounts.AppData which +* is part of the larger VNLib collection of libraries and utilities. +* +* VNLib.Plugins.Essentials.Accounts is free software: you can redistribute it and/or modify +* it under the terms of the GNU Affero General Public License as +* published by the Free Software Foundation, either version 3 of the +* License, or (at your option) any later version. +* +* VNLib.Plugins.Essentials.Accounts is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Affero General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see https://www.gnu.org/licenses/. +*/ + +using System; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; + +using Microsoft.EntityFrameworkCore; + +using VNLib.Utils.Logging; +using VNLib.Plugins.Extensions.Loading; +using VNLib.Plugins.Extensions.Loading.Sql; +using VNLib.Plugins.Extensions.Data; +using VNLib.Plugins.Extensions.Data.Abstractions; +using VNLib.Plugins.Extensions.Data.Extensions; +using VNLib.Plugins.Essentials.Accounts.AppData.Model; + +namespace VNLib.Plugins.Essentials.Accounts.AppData.Stores.Sql +{ + + internal sealed class SqlBackingStore(PluginBase plugin) : IAppDataStore, IAsyncConfigurable + { + private readonly DbRecordStore _store = new(plugin.GetContextOptionsAsync()); + + ///<inheritdoc/> + async Task IAsyncConfigurable.ConfigureServiceAsync(PluginBase plugin) + { + //Wait for the options to be ready + await _store.WhenLoaded(); + + //Add startup delay + await Task.Delay(2000); + + plugin.Log.Debug("Creating database tables for Account AppData"); + + await plugin.EnsureDbCreatedAsync<UserRecordDbContext>(plugin); + } + + ///<inheritdoc/> + public Task DeleteRecordAsync(string userId, string recordKey, CancellationToken cancellation) + { + return _store.DeleteAsync([userId, recordKey], cancellation); + } + + ///<inheritdoc/> + public async Task<UserRecordData?> GetRecordAsync(string userId, string recordKey, RecordOpFlags flags, CancellationToken cancellation) + { + DataRecord? dr = await _store.GetSingleAsync(userId, recordKey); + + if (dr is null) + { + return null; + } + + //get the last modified time in unix time for the caller + long lastModifed = new DateTimeOffset(dr.LastModified).ToUnixTimeSeconds(); + + return new(userId, dr.Data!, lastModifed, unchecked((ulong)dr.Checksum)); + } + + ///<inheritdoc/> + public Task SetRecordAsync(string userId, string recordKey, byte[] data, ulong checksum, RecordOpFlags flags, CancellationToken cancellation) + { + return _store.AddOrUpdateAsync(new DataRecord + { + UserId = userId, + RecordKey = recordKey, + Data = data, + Checksum = unchecked((long)checksum) + }, cancellation); + } + + sealed class DbRecordStore(IAsyncLazy<DbContextOptions> options) : DbStore<DataRecord> + { + public async Task WhenLoaded() => await options; + + ///<inheritdoc/> + public override IDbQueryLookup<DataRecord> QueryTable { get; } = new DbQueries(); + + ///<inheritdoc/> + public override IDbContextHandle GetNewContext() => new UserRecordDbContext(options.Value); + + ///<inheritdoc/> + public override string GetNewRecordId() => Guid.NewGuid().ToString("N"); + + ///<inheritdoc/> + public override void OnRecordUpdate(DataRecord newRecord, DataRecord existing) + { + existing.Data = newRecord.Data; + existing.Checksum = newRecord.Checksum; + existing.RecordKey = newRecord.RecordKey; + existing.UserId = newRecord.UserId; + existing.Created = newRecord.Created; + } + + sealed class DbQueries : IDbQueryLookup<DataRecord> + { + public IQueryable<DataRecord> GetCollectionQueryBuilder(IDbContextHandle context, params string[] constraints) + { + throw new NotSupportedException("Lists for users is not queryable. Callers must submit a record key"); + } + + public IQueryable<DataRecord> GetSingleQueryBuilder(IDbContextHandle context, params string[] constraints) + { + string userId = constraints[0]; + string recordKey = constraints[1]; + + return from r in context.Set<DataRecord>() + where r.UserId == userId && r.RecordKey == recordKey + select r; + } + + public IQueryable<DataRecord> AddOrUpdateQueryBuilder(IDbContextHandle context, DataRecord record) + { + return GetSingleQueryBuilder(context, record.UserId!, record.RecordKey!); + } + } + } + + + } +} diff --git a/plugins/VNLib.Plugins.Essentials.Accounts.AppData/src/Stores/Sql/UserRecordDbContext.cs b/plugins/VNLib.Plugins.Essentials.Accounts.AppData/src/Stores/Sql/UserRecordDbContext.cs new file mode 100644 index 0000000..1ab8767 --- /dev/null +++ b/plugins/VNLib.Plugins.Essentials.Accounts.AppData/src/Stores/Sql/UserRecordDbContext.cs @@ -0,0 +1,59 @@ +/* +* Copyright (c) 2024 Vaughn Nugent +* +* Library: VNLib +* Package: VNLib.Plugins.Essentials.Accounts.AppData +* File: UserRecordDbContext.cs +* +* UserRecordDbContext.cs is part of VNLib.Plugins.Essentials.Accounts.AppData which +* is part of the larger VNLib collection of libraries and utilities. +* +* VNLib.Plugins.Essentials.Accounts is free software: you can redistribute it and/or modify +* it under the terms of the GNU Affero General Public License as +* published by the Free Software Foundation, either version 3 of the +* License, or (at your option) any later version. +* +* VNLib.Plugins.Essentials.Accounts is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Affero General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see https://www.gnu.org/licenses/. +*/ + +using Microsoft.EntityFrameworkCore; + +using VNLib.Plugins.Extensions.Data; +using VNLib.Plugins.Extensions.Loading.Sql; + +namespace VNLib.Plugins.Essentials.Accounts.AppData.Stores.Sql +{ + internal sealed class UserRecordDbContext : DBContextBase, IDbTableDefinition + { + public DbSet<DataRecord> UserDataRecords { get; set; } + + public UserRecordDbContext(DbContextOptions options) : base(options) + { } + + public UserRecordDbContext() + { } + + public void OnDatabaseCreating(IDbContextBuilder builder, object? userState) + { + //Define the table for the data records + builder.DefineTable<DataRecord>(nameof(UserDataRecords), table => + { + //Define table columns + table.WithColumn(p => p.Id).AllowNull(false); + table.WithColumn(p => p.Version).TimeStamp(); + table.WithColumn(p => p.RecordKey).AllowNull(false); + table.WithColumn(p => p.UserId).AllowNull(false); + table.WithColumn(p => p.Created); + table.WithColumn(p => p.LastModified); + table.WithColumn(p => p.Data); + table.WithColumn(p => p.Checksum); + }); + } + } +} diff --git a/plugins/VNLib.Plugins.Essentials.Accounts.AppData/src/VNLib.Plugins.Essentials.Accounts.AppData.csproj b/plugins/VNLib.Plugins.Essentials.Accounts.AppData/src/VNLib.Plugins.Essentials.Accounts.AppData.csproj new file mode 100644 index 0000000..580faba --- /dev/null +++ b/plugins/VNLib.Plugins.Essentials.Accounts.AppData/src/VNLib.Plugins.Essentials.Accounts.AppData.csproj @@ -0,0 +1,67 @@ +<Project Sdk="Microsoft.NET.Sdk"> + + <PropertyGroup> + <Nullable>enable</Nullable> + <TargetFramework>net8.0</TargetFramework> + <RootNamespace>VNLib.Plugins.Essentials.Accounts.AppData</RootNamespace> + <AssemblyName>Essentials.AppData</AssemblyName> + <AnalysisLevel>latest-all</AnalysisLevel> + <NeutralLanguage>en-US</NeutralLanguage> + <GenerateDocumentationFile>true</GenerateDocumentationFile> + <!--Enable dynamic loading--> + <EnableDynamicLoading>true</EnableDynamicLoading> + </PropertyGroup> + + <PropertyGroup> + <PackageId>VNLib.Plugins.Essentials.Accounts.AppData</PackageId> + <Authors>Vaughn Nugent</Authors> + <Company>Vaughn Nugent</Company> + <Product>Essentials Account-Based client data storage</Product> + <Copyright>Copyright © 2024 Vaughn Nugent</Copyright> + <PackageProjectUrl>https://www.vaughnnugent.com/resources/software/modules/Plugins.Essentials</PackageProjectUrl> + <RepositoryUrl>https://github.com/VnUgE/Plugins.Essentials/tree/master/plugins/VNLib.Plugins.Essentials.Accounts.AppData</RepositoryUrl> + <Description>An Essentials plugin that provides endpoints for web-application synchronized storage such as user preferences</Description> + </PropertyGroup> + + <PropertyGroup> + <PackageReadmeFile>README.md</PackageReadmeFile> + <PackageLicenseFile>LICENSE</PackageLicenseFile> + </PropertyGroup> + + <ItemGroup> + <None Include="..\..\..\LICENSE"> + <Pack>True</Pack> + <PackagePath>\</PackagePath> + <CopyToOutputDirectory>Always</CopyToOutputDirectory> + </None> + <None Include="..\README.md"> + <Pack>True</Pack> + <PackagePath>\</PackagePath> + </None> + </ItemGroup> + + <ItemGroup> + <PackageReference Include="ErrorProne.NET.CoreAnalyzers" Version="0.1.2"> + <PrivateAssets>all</PrivateAssets> + <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets> + </PackageReference> + <PackageReference Include="ErrorProne.NET.Structs" Version="0.1.2"> + <PrivateAssets>all</PrivateAssets> + <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets> + </PackageReference> + <PackageReference Include="MemoryPack" Version="1.20.3" /> + </ItemGroup> + + <ItemGroup> + <ProjectReference Include="..\..\..\..\VNLib.Data.Caching\lib\VNLib.Plugins.Extensions.VNCache\src\VNLib.Plugins.Extensions.VNCache.csproj" /> + <ProjectReference Include="..\..\..\..\VNLib.Plugins.Extensions\lib\VNLib.Plugins.Extensions.Data\src\VNLib.Plugins.Extensions.Data.csproj" /> + <ProjectReference Include="..\..\..\..\VNLib.Plugins.Extensions\lib\VNLib.Plugins.Extensions.Loading.Sql\src\VNLib.Plugins.Extensions.Loading.Sql.csproj" /> + <ProjectReference Include="..\..\..\..\VNLib.Plugins.Extensions\lib\VNLib.Plugins.Extensions.Validation\src\VNLib.Plugins.Extensions.Validation.csproj" /> + </ItemGroup> + + + <Target Condition="'$(BuildingInsideVisualStudio)' == true" Name="PostBuild" AfterTargets="PostBuildEvent"> + <Exec Command="start xcopy "$(TargetDir)" "$(SolutionDir)devplugins\$(TargetName)" /E /Y /R" /> + </Target> + +</Project> |