feat: Update caching with new helpers

7 files changed, 301 insertions, 389 deletions

diff --git a/plugins/VNLib.Plugins.Essentials.Accounts.AppData/src/CacheStore.cs b/plugins/VNLib.Plugins.Essentials.Accounts.AppData/src/CacheStore.cs
deleted file mode 100644
index 95c1b5a..0000000
--- a/plugins/VNLib.Plugins.Essentials.Accounts.AppData/src/CacheStore.cs
+++ /dev/null
@@ -1,249 +0,0 @@
-/*
-* Copyright (c) 2024 Vaughn Nugent
-* 
-* Library: VNLib
-* Package: VNLib.Plugins.Essentials.Accounts.AppData
-* File: CacheStore.cs 
-*
-* CacheStore.cs is part of VNLib.Plugins.Essentials.Accounts.AppData which 
-* is part of the larger VNLib collection of libraries and utilities.
-*
-* VNLib.Plugins.Essentials.Accounts is free software: you can redistribute it and/or modify 
-* it under the terms of the GNU Affero General Public License as 
-* published by the Free Software Foundation, either version 3 of the
-* License, or (at your option) any later version.
-*
-* VNLib.Plugins.Essentials.Accounts is distributed in the hope that it will be useful,
-* but WITHOUT ANY WARRANTY; without even the implied warranty of
-* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-* GNU Affero General Public License for more details.
-*
-* You should have received a copy of the GNU Affero General Public License
-* along with this program.  If not, see https://www.gnu.org/licenses/.
-*/
-
-using System;
-using System.Buffers;
-using System.Threading;
-using System.Threading.Tasks;
-
-using MemoryPack;
-
-using VNLib.Utils.Extensions;
-using VNLib.Utils.Logging;
-using VNLib.Hashing.Checksums;
-using VNLib.Data.Caching;
-using VNLib.Plugins.Extensions.Loading;
-using VNLib.Plugins.Extensions.VNCache;
-using VNLib.Plugins.Extensions.VNCache.DataModel;
-using VNLib.Plugins.Essentials.Accounts.AppData.Stores;
-using VNLib.Plugins.Essentials.Accounts.AppData.Model;
-
-namespace VNLib.Plugins.Essentials.Accounts.AppData
-{
-
-    [ConfigurationName("record_cache")]
-    internal sealed class CacheStore : IAppDataStore
-    {
-        const string LogScope = "Record Cache";
-
-        private readonly IEntityCache<RecordDataCacheEntry> _cache;
-        private readonly PersistentStorageManager _backingStore;
-        private readonly ILogProvider _logger;
-        private readonly bool AlwaysObserverCacheUpdate;
-        private readonly TimeSpan CacheTTL;
-
-
-        public CacheStore(PluginBase plugin, IConfigScope config)
-        {
-            string cachePrefix = config.GetRequiredProperty("prefix", p => p.GetString()!);
-            CacheTTL = config.GetRequiredProperty("ttl", p => p.GetTimeSpan(TimeParseType.Seconds))!;
-            AlwaysObserverCacheUpdate = config.GetRequiredProperty("force_write_through", p => p.GetBoolean())!;
-            _logger = plugin.Log.CreateScope(LogScope);
-
-            //Load persistent storage manager
-            _backingStore = plugin.GetOrCreateSingleton<PersistentStorageManager>();
-
-            //Use memory pack for serialization
-            MpSerializer serializer = new();
-
-            /*
-             * Initialize entity cache from the default global cache provider,
-             * then create a prefixed cache for the app data records.
-             * 
-             * The app should make sure that the cache provider is available
-             * otherwise do not load this component.
-             */
-            _cache = plugin.GetDefaultGlobalCache()
-                ?.GetPrefixedCache(cachePrefix)
-                ?.CreateEntityCache<RecordDataCacheEntry>(serializer, serializer)
-                ?? throw new InvalidOperationException("No cache provider is available");
-
-            _logger.Verbose("Cache and backing store initialized");
-        }
-
-        ///<inheritdoc/>
-        public Task DeleteRecordAsync(string userId, string recordKey, CancellationToken cancellation)
-        {
-            /*
-             * Deleting entires does not matter if they existed previously or not. Just 
-             * that the opeation executed successfully. 
-             * 
-             * Parallelize the delete operation to the cache and the backing store
-             */
-            Task fromCache = _cache.RemoveAsync(GetCacheKey(userId, recordKey), cancellation);
-            Task fromDb = _backingStore.DeleteRecordAsync(userId, recordKey, cancellation);
-
-            return Task.WhenAll(fromCache, fromDb);
-        }
-
-        ///<inheritdoc/>
-        public async Task<UserRecordData?> GetRecordAsync(string userId, string recordKey, RecordOpFlags flags, CancellationToken cancellation)
-        {
-            bool useCache = (flags & RecordOpFlags.NoCache) == 0;
-
-            //See if caller wants to bypass cache
-            if (useCache)
-            {
-                string cacheKey = GetCacheKey(userId, recordKey);
-
-                //try fetching from cache
-                RecordDataCacheEntry? cached = await _cache.GetAsync(cacheKey, cancellation);
-
-                //if cache is valid, return it
-                if (cached != null && !IsCacheExpired(cached))
-                {
-                    return new(userId, cached.RecordData, cached.UnixTimestamp, cached.Checksum);
-                }
-            }
-
-            //fetch from db
-            UserRecordData? stored = await _backingStore.GetRecordAsync(userId, recordKey, flags, cancellation);
-
-            //If the record is valid and cache is enabled, update the record in cache
-            if (useCache && stored is not null)
-            {
-                //If no checksum is present, calculate it before storing in cache
-                if (!stored.Checksum.HasValue)
-                {
-                    ulong checksum = FNV1a.Compute64(stored.Data);
-                    stored = stored with { Checksum = checksum };
-                }
-
-                //update cached version
-                Task update = DeferCacheUpdate(
-                    userId,
-                    recordKey,
-                    stored.Data,
-                    stored.LastModifed,
-                    stored.Checksum.Value
-                );
-
-                if (AlwaysObserverCacheUpdate || (flags & RecordOpFlags.WriteThrough) != 0)
-                {
-                    //Wait for cache update to complete
-                    await update.ConfigureAwait(false);
-                }
-                else
-                {
-                    //Defer the cache update and continue
-                    WatchDeferredCacheUpdate(update);
-                }
-            }
-
-            return stored;
-        }
-
-        ///<inheritdoc/>
-        public Task SetRecordAsync(string userId, string recordKey, byte[] data, ulong checksum, RecordOpFlags flags, CancellationToken cancellation)
-        {
-
-            //Always push update to db
-            Task db = _backingStore.SetRecordAsync(userId, recordKey, data, checksum, flags, cancellation);
-
-            //Optionally push update to cache
-            Task cache = Task.CompletedTask;
-
-            if ((flags & RecordOpFlags.NoCache) == 0)
-            {
-                long time = DateTimeOffset.Now.ToUnixTimeSeconds();
-
-                //Push update to cache
-                cache = DeferCacheUpdate(userId, recordKey, data, time, checksum);
-            }
-
-            /*
-             * If writethough is not set, updates will always be deferred 
-             * and this call will return immediately.
-             * 
-             * We still need to observe the task incase an error occurs
-             */
-            Task all = Task.WhenAll(db, cache);
-
-            if (AlwaysObserverCacheUpdate || (flags & RecordOpFlags.WriteThrough) != 0)
-            {
-                return all;
-            }
-            else
-            {
-                WatchDeferredCacheUpdate(all);
-                return Task.CompletedTask;
-            }
-        }
-
-        private string GetCacheKey(string userId, string recordKey) => $"{userId}:{recordKey}";
-
-        private bool IsCacheExpired(RecordDataCacheEntry entry)
-        {
-            return DateTimeOffset.FromUnixTimeSeconds(entry.UnixTimestamp).Add(CacheTTL) < DateTimeOffset.Now;
-        }
-
-        private Task DeferCacheUpdate(string userId, string recordKey, byte[] data, long time, ulong checksum)
-        {
-            string cacheKey = GetCacheKey(userId, recordKey);
-
-            RecordDataCacheEntry entry = new()
-            {
-                Checksum = checksum,
-                RecordData = data,
-                UnixTimestamp = time
-            };
-
-            return _cache.UpsertAsync(cacheKey, entry);
-        }
-
-        private async void WatchDeferredCacheUpdate(Task update)
-        {
-            try
-            {
-                await update.ConfigureAwait(false);
-            }
-            catch (Exception e)
-            {
-                if (_logger.IsEnabled(LogLevel.Debug))
-                {
-                    _logger.Warn(e, "Failed to update cached User AppData record");
-                }
-                else
-                {
-                    _logger.Warn("Failed to update cached AppData record");
-                }
-            }
-        }
-
-
-        private sealed class MpSerializer : ICacheObjectDeserializer, ICacheObjectSerializer
-        {
-
-            public T? Deserialize<T>(ReadOnlySpan<byte> objectData)
-            {
-                return MemoryPackSerializer.Deserialize<T>(objectData);
-            }
-
-            public void Serialize<T>(T obj, IBufferWriter<byte> finiteWriter)
-            {
-                MemoryPackSerializer.Serialize(finiteWriter, obj);
-            }
-        }
-    }
-}
diff --git a/plugins/VNLib.Plugins.Essentials.Accounts.AppData/src/Endpoints/WebEndpoint.cs b/plugins/VNLib.Plugins.Essentials.Accounts.AppData/src/Endpoints/WebEndpoint.cs
index 9c8f501..b95930d 100644
--- a/plugins/VNLib.Plugins.Essentials.Accounts.AppData/src/Endpoints/WebEndpoint.cs
+++ b/plugins/VNLib.Plugins.Essentials.Accounts.AppData/src/Endpoints/WebEndpoint.cs
@@ -28,11 +28,9 @@ using System.Collections.Generic;
 using System.Threading.Tasks;
 
 using VNLib.Net.Http;
-using VNLib.Utils.Logging;
 using VNLib.Hashing.Checksums;
 using VNLib.Plugins.Essentials.Endpoints;
 using VNLib.Plugins.Essentials.Extensions;
-using VNLib.Plugins.Extensions.VNCache;
 using VNLib.Plugins.Extensions.Loading;
 using VNLib.Plugins.Extensions.Validation;
 
@@ -46,42 +44,19 @@ namespace VNLib.Plugins.Essentials.Accounts.AppData.Endpoints
     {
         const int DefaultMaxDataSize = 8 * 1024;
 
-        private readonly IAppDataStore _store;
+        private readonly StorageManager _store;
         private readonly int MaxDataSize;
         private readonly string[] AllowedScopes;
 
         public WebEndpoint(PluginBase plugin, IConfigScope config)
         {
-            string path = config.GetRequiredProperty("path", p => p.GetString())!;
+            string path = config.GetRequiredProperty<string>("path");
             InitPathAndLog(path, plugin.Log.CreateScope("Endpoint"));
 
-            MaxDataSize = config.GetValueOrDefault("max_data_size", p => p.GetInt32(), DefaultMaxDataSize);
-            AllowedScopes = config.GetRequiredProperty("allowed_scopes", p => p.EnumerateArray().Select(p => p.GetString()!)).ToArray();
-
-            bool useCache = false;
-
-            //Cache loading is optional
-            if (plugin.HasConfigForType<CacheStore>())
-            {
-                //See if caching is enabled
-                IConfigScope cacheConfig = plugin.GetConfigForType<CacheStore>();
-                useCache = cacheConfig.GetValueOrDefault("enabled", e => e.GetBoolean(), false);
-
-                if (useCache && plugin.GetDefaultGlobalCache() is null)
-                {
-                    plugin.Log.Error("Cache was enabled but no caching library was loaded. Continuing without cache");
-                    useCache = false;
-                }
-            }
-
-            _store = LoadStore(plugin, useCache);
-        }
-
-        private static IAppDataStore LoadStore(PluginBase plugin, bool withCache)
-        {
-            return withCache 
-                ? plugin.GetOrCreateSingleton<CacheStore>() 
-                : plugin.GetOrCreateSingleton<PersistentStorageManager>();
+            MaxDataSize = config.GetValueOrDefault("max_data_size", DefaultMaxDataSize);
+            AllowedScopes = config.GetRequiredProperty<string[]>("allowed_scopes");
+         
+            _store = plugin.GetOrCreateSingleton<StorageManager>();
         }
 
         protected async override ValueTask<VfReturnType> GetAsync(HttpEntity entity)
diff --git a/plugins/VNLib.Plugins.Essentials.Accounts.AppData/src/Model/RecordDataCacheEntry.cs b/plugins/VNLib.Plugins.Essentials.Accounts.AppData/src/Model/AppDataRequest.cs
index 9c0767d..f61bdef 100644
--- a/plugins/VNLib.Plugins.Essentials.Accounts.AppData/src/Model/RecordDataCacheEntry.cs
+++ b/plugins/VNLib.Plugins.Essentials.Accounts.AppData/src/Model/AppDataRequest.cs
@@ -3,9 +3,9 @@
 * 
 * Library: VNLib
 * Package: VNLib.Plugins.Essentials.Accounts.AppData
-* File: RecordDataCacheEntry.cs 
+* File: IAppDataStore.cs 
 *
-* RecordDataCacheEntry.cs is part of VNLib.Plugins.Essentials.Accounts.AppData which 
+* IAppDataStore.cs is part of VNLib.Plugins.Essentials.Accounts.AppData which 
 * is part of the larger VNLib collection of libraries and utilities.
 *
 * VNLib.Plugins.Essentials.Accounts is free software: you can redistribute it and/or modify 
@@ -22,17 +22,18 @@
 * along with this program.  If not, see https://www.gnu.org/licenses/.
 */
 
-using MemoryPack;
+using System;
+
+using VNLib.Plugins.Extensions.VNCache.DataModel;
 
 namespace VNLib.Plugins.Essentials.Accounts.AppData.Model
 {
-    [MemoryPackable]
-    internal partial class RecordDataCacheEntry
+    internal sealed record class AppDataRequest(string UserId, string RecordKey) : IEntityCacheKey
     {
-        public byte[] RecordData { get; set; }
-
-        public ulong? Checksum { get; set; }
+        ///<inheritdoc/>
+        public string GetKey() => $"{UserId}:{RecordKey}";
 
-        public long UnixTimestamp { get; set; }
+        ///<inheritdoc/>
+        public override int GetHashCode() => HashCode.Combine(UserId, RecordKey);
     }
 }
diff --git a/plugins/VNLib.Plugins.Essentials.Accounts.AppData/src/Model/UserRecordData.cs b/plugins/VNLib.Plugins.Essentials.Accounts.AppData/src/Model/UserRecordData.cs
index d3770c6..5ee5da3 100644
--- a/plugins/VNLib.Plugins.Essentials.Accounts.AppData/src/Model/UserRecordData.cs
+++ b/plugins/VNLib.Plugins.Essentials.Accounts.AppData/src/Model/UserRecordData.cs
@@ -22,7 +22,17 @@
 * along with this program.  If not, see https://www.gnu.org/licenses/.
 */
 
+using MemoryPack;
+
 namespace VNLib.Plugins.Essentials.Accounts.AppData.Model
 {
-    internal record class UserRecordData(string UserId, byte[] Data, long LastModifed, ulong? Checksum);
+    [MemoryPackable]
+    internal sealed partial record class UserRecordData
+    {
+        public required byte[] Data { get; init; }
+
+        public required ulong? Checksum { get; init; }
+
+        internal long CacheTimestamp { get; set; }
+    }
 }
diff --git a/plugins/VNLib.Plugins.Essentials.Accounts.AppData/src/Stores/PersistentStorageManager.cs b/plugins/VNLib.Plugins.Essentials.Accounts.AppData/src/Stores/PersistentStorageManager.cs
deleted file mode 100644
index 99a3286..0000000
--- a/plugins/VNLib.Plugins.Essentials.Accounts.AppData/src/Stores/PersistentStorageManager.cs
+++ /dev/null
@@ -1,75 +0,0 @@
-/*
-* Copyright (c) 2024 Vaughn Nugent
-* 
-* Library: VNLib
-* Package: VNLib.Plugins.Essentials.Accounts.AppData
-* File: PersistentStorageManager.cs 
-*
-* PersistentStorageManager.cs is part of VNLib.Plugins.Essentials.Accounts.AppData which 
-* is part of the larger VNLib collection of libraries and utilities.
-*
-* VNLib.Plugins.Essentials.Accounts is free software: you can redistribute it and/or modify 
-* it under the terms of the GNU Affero General Public License as 
-* published by the Free Software Foundation, either version 3 of the
-* License, or (at your option) any later version.
-*
-* VNLib.Plugins.Essentials.Accounts is distributed in the hope that it will be useful,
-* but WITHOUT ANY WARRANTY; without even the implied warranty of
-* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-* GNU Affero General Public License for more details.
-*
-* You should have received a copy of the GNU Affero General Public License
-* along with this program.  If not, see https://www.gnu.org/licenses/.
-*/
-
-using System;
-using System.Threading;
-using System.Threading.Tasks;
-
-using VNLib.Utils.Logging;
-using VNLib.Plugins.Extensions.Loading;
-
-using VNLib.Plugins.Essentials.Accounts.AppData.Model;
-using VNLib.Plugins.Essentials.Accounts.AppData.Stores.Sql;
-
-namespace VNLib.Plugins.Essentials.Accounts.AppData.Stores
-{
-    [ConfigurationName("storage")]
-    internal sealed class PersistentStorageManager : IAppDataStore
-    {
-        private readonly IAppDataStore _backingStore;
-
-        public PersistentStorageManager(PluginBase plugin, IConfigScope config)
-        {
-            string storeType = config.GetRequiredProperty("type", p => p.GetString()!).ToLower(null);
-
-            switch (storeType)
-            {
-                case "sql":
-                    _backingStore = plugin.GetOrCreateSingleton<SqlBackingStore>();
-                    plugin.Log.Information("Using SQL based backing store");
-                    break;
-                default:
-                    throw new NotSupportedException($"Storage type {storeType} is not supported");
-            }
-        }
-
-        ///<inheritdoc/>
-        public Task DeleteRecordAsync(string userId, string recordKey, CancellationToken cancellation)
-        {
-            return _backingStore.DeleteRecordAsync(userId, recordKey, cancellation);
-        }
-
-        ///<inheritdoc/>
-        public Task<UserRecordData?> GetRecordAsync(string userId, string recordKey, RecordOpFlags flags, CancellationToken cancellation)
-        {
-            return _backingStore.GetRecordAsync(userId, recordKey, flags, cancellation);
-        }
-
-        ///<inheritdoc/>
-        public Task SetRecordAsync(string userId, string recordKey, byte[] data, ulong checksum, RecordOpFlags flags, CancellationToken cancellation)
-        {
-            return _backingStore.SetRecordAsync(userId, recordKey, data, checksum, flags, cancellation);
-        }
-    }
-}
diff --git a/plugins/VNLib.Plugins.Essentials.Accounts.AppData/src/Stores/Sql/SqlBackingStore.cs b/plugins/VNLib.Plugins.Essentials.Accounts.AppData/src/Stores/Sql/SqlBackingStore.cs
index f67c652..0281a0b 100644
--- a/plugins/VNLib.Plugins.Essentials.Accounts.AppData/src/Stores/Sql/SqlBackingStore.cs
+++ b/plugins/VNLib.Plugins.Essentials.Accounts.AppData/src/Stores/Sql/SqlBackingStore.cs
@@ -29,47 +29,37 @@ using System.Threading.Tasks;
 
 using Microsoft.EntityFrameworkCore;
 
-using VNLib.Utils.Logging;
+using VNLib.Utils;
 using VNLib.Plugins.Extensions.Loading;
 using VNLib.Plugins.Extensions.Loading.Sql;
 using VNLib.Plugins.Extensions.Data;
 using VNLib.Plugins.Extensions.Data.Abstractions;
 using VNLib.Plugins.Extensions.Data.Extensions;
+using VNLib.Plugins.Extensions.VNCache.DataModel;
+
 using VNLib.Plugins.Essentials.Accounts.AppData.Model;
 
 namespace VNLib.Plugins.Essentials.Accounts.AppData.Stores.Sql
 {
 
-    internal sealed class SqlBackingStore(PluginBase plugin) : IAppDataStore, IAsyncConfigurable
+    internal sealed class SqlBackingStore(PluginBase plugin) : IEntityStore<UserRecordData, AppDataRequest>, IAsyncConfigurable
     {
         private readonly DbRecordStore _store = new(plugin.GetContextOptionsAsync());
 
         ///<inheritdoc/>
         async Task IAsyncConfigurable.ConfigureServiceAsync(PluginBase plugin)
         {
-            //Wait for the options to be ready
-            await _store.WhenLoaded();
-
             //Add startup delay
             await Task.Delay(2000);
-
-            plugin.Log.Debug("Creating database tables for Account AppData");
-
             await plugin.EnsureDbCreatedAsync<UserRecordDbContext>(plugin);
         }
-
-        ///<inheritdoc/>
-        public Task DeleteRecordAsync(string userId, string recordKey, CancellationToken cancellation)
-        {
-            return _store.DeleteAsync([userId, recordKey], cancellation);
-        }
-
+    
         ///<inheritdoc/>
-        public async Task<UserRecordData?> GetRecordAsync(string userId, string recordKey, RecordOpFlags flags, CancellationToken cancellation)
+        public async Task<UserRecordData?> GetAsync(AppDataRequest request, CancellationToken cancellation = default)
         {
-            DataRecord? dr = await _store.GetSingleAsync(userId, recordKey);
+            DataRecord? dr = await _store.GetSingleAsync([request.UserId, request.RecordKey]);
 
-            if (dr is null)
+            if (dr is null || !string.Equals(dr.UserId, request.UserId, StringComparison.Ordinal))
             {
                 return null;
             }
@@ -77,21 +67,37 @@ namespace VNLib.Plugins.Essentials.Accounts.AppData.Stores.Sql
             //get the last modified time in unix time for the caller
             long lastModifed = new DateTimeOffset(dr.LastModified).ToUnixTimeSeconds();
 
-            return new(userId, dr.Data!, lastModifed, unchecked((ulong)dr.Checksum));
+            return new()
+            {
+                Data            = dr.Data,
+                CacheTimestamp  = lastModifed,
+                Checksum        = dr.Checksum == 0 ? null : unchecked((uint)dr.Checksum)
+            };
         }
 
         ///<inheritdoc/>
-        public Task SetRecordAsync(string userId, string recordKey, byte[] data, ulong checksum, RecordOpFlags flags, CancellationToken cancellation)
+        public Task UpsertAsync(AppDataRequest request, UserRecordData entity, CancellationToken cancellation = default)
         {
             return _store.AddOrUpdateAsync(new DataRecord
             {
-                UserId = userId,
-                RecordKey = recordKey,
-                Data = data,
-                Checksum = unchecked((long)checksum)
+                UserId = request.UserId,
+                RecordKey = request.RecordKey,
+                Data = entity.Data,
+                Checksum = entity.Checksum.HasValue ? unchecked((long)entity.Checksum.Value) : 0,
+
             }, cancellation);
         }
 
+        ///<inheritdoc/>
+        public async Task<bool> RemoveAsync(AppDataRequest request, CancellationToken cancellation = default)
+        {
+            ERRNO result = await _store.DeleteAsync([request.UserId, request.RecordKey], cancellation)
+                .ConfigureAwait(false);
+
+            return result > 0;
+        }
+
+
         sealed class DbRecordStore(IAsyncLazy<DbContextOptions> options) : DbStore<DataRecord>
         {
             public async Task WhenLoaded() => await options;
diff --git a/plugins/VNLib.Plugins.Essentials.Accounts.AppData/src/Stores/StorageManager.cs b/plugins/VNLib.Plugins.Essentials.Accounts.AppData/src/Stores/StorageManager.cs
new file mode 100644
index 0000000..bbfe9c6
--- /dev/null
+++ b/plugins/VNLib.Plugins.Essentials.Accounts.AppData/src/Stores/StorageManager.cs
@@ -0,0 +1,244 @@
+/*
+* Copyright (c) 2024 Vaughn Nugent
+* 
+* Library: VNLib
+* Package: VNLib.Plugins.Essentials.Accounts.AppData
+* File: PersistentStorageManager.cs 
+*
+* PersistentStorageManager.cs is part of VNLib.Plugins.Essentials.Accounts.AppData which 
+* is part of the larger VNLib collection of libraries and utilities.
+*
+* VNLib.Plugins.Essentials.Accounts is free software: you can redistribute it and/or modify 
+* it under the terms of the GNU Affero General Public License as 
+* published by the Free Software Foundation, either version 3 of the
+* License, or (at your option) any later version.
+*
+* VNLib.Plugins.Essentials.Accounts is distributed in the hope that it will be useful,
+* but WITHOUT ANY WARRANTY; without even the implied warranty of
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+* GNU Affero General Public License for more details.
+*
+* You should have received a copy of the GNU Affero General Public License
+* along with this program.  If not, see https://www.gnu.org/licenses/.
+*/
+
+using System;
+using System.Buffers;
+using System.Threading;
+using System.Threading.Tasks;
+using System.Text.Json.Serialization;
+
+using MemoryPack;
+
+using VNLib.Hashing;
+using VNLib.Utils.Logging;
+using VNLib.Data.Caching;
+using VNLib.Plugins.Extensions.Loading;
+using VNLib.Plugins.Essentials.Accounts.AppData.Model;
+using VNLib.Plugins.Essentials.Accounts.AppData.Stores.Sql;
+using VNLib.Plugins.Extensions.VNCache;
+using VNLib.Plugins.Extensions.VNCache.DataModel;
+
+
+namespace VNLib.Plugins.Essentials.Accounts.AppData.Stores
+{
+    [ConfigurationName("storage")]
+    internal sealed class StorageManager : IAppDataStore
+    {
+        private readonly SqlBackingStore _backingStore;
+        private readonly EntityResultCache<UserRecordData>? _cache;
+        private readonly ILogProvider _logger;
+
+        public StorageManager(PluginBase plugin, IConfigScope config)
+        {
+            string storeType = config.GetRequiredProperty<string>("type").ToLower(null);
+
+            _logger = plugin.Log.CreateScope("STORE");
+
+            switch (storeType)
+            {
+                case "sql":
+                    _backingStore = plugin.GetOrCreateSingleton<SqlBackingStore>();
+                    plugin.Log.Information("Using SQL based backing store");
+                    break;
+                default:
+                    throw new NotSupportedException($"Storage type {storeType} is not supported");
+            }
+
+            CacheConfig? cConfig = config.GetValueOrDefault<CacheConfig?>("cache", defaultValue: null);
+
+            if(cConfig is null || cConfig.Enabled == false)
+            {
+                _logger.Debug("Result cache disabled via configuration, or not set");
+                return;
+            }
+
+            IGlobalCacheProvider? cache = plugin.GetDefaultGlobalCache();
+
+            if (cache is null)
+            {
+                _logger.Warn("Cache was enabled, but no global cache library was loaded. Caching disabled");
+                return;
+            }
+
+            /*
+             * When using a shared global cache, prefixing keys is important to avoid
+             * key collisions with other plugins. It is also a security measure to prevent
+             * other systems from reading senstive data with any type of key injection
+             * from other systems like sessions, or reading sessions from this plugin 
+             * and so on.
+             * 
+             * A static prefix should be used and shared between servers for optimal
+             * cache performance. If a prefix is not set, a random prefix will be generated
+             * and logged to the console.
+             */
+            if (string.IsNullOrWhiteSpace(cConfig.Prefix))
+            {              
+                cConfig.Prefix = RandomHash.GetRandomBase32(8);
+                _logger.Warn("CACHE: No prefix was set, using random prefix: {prefix}", cConfig.Prefix);
+            }
+
+            MpSerializer serializer = new();
+            MpCachePolicy expPolicy = new(TimeSpan.FromSeconds(cConfig.CacheTTL));
+
+            ICacheTaskPolicy cacheTaskPolicy = cConfig.WriteBack
+                ? new WriteBackCachePolicy(OnTaskError)
+                : WriteThroughCachePolicy.Instance;
+
+            _cache = cache
+                .GetPrefixedCache(cConfig.Prefix)
+                .CreateEntityCache<UserRecordData>(serializer, serializer)
+                .CreateResultCache()
+                .WithExpirationPoicy(expPolicy)
+                .WithTaskPolicy(cacheTaskPolicy)
+                .Build();
+        }
+      
+        public Task DeleteRecordAsync(string userId, string recordKey, CancellationToken cancellation)
+        {
+            AppDataRequest adr = new (userId, recordKey);
+
+            //Attempt to purge from cache and store in parallel if cache is enabled
+            Task cacheDel = _cache is not null 
+                ? _cache.RemoveAsync(userId, cancellation)
+                : Task.CompletedTask;
+
+            Task storeRemove = _backingStore.RemoveAsync(adr, cancellation);
+
+            return Task.WhenAll(cacheDel, storeRemove);
+        }
+     
+        ///<inheritdoc/>
+        public Task<UserRecordData?> GetRecordAsync(string userId, string recordKey, RecordOpFlags flags, CancellationToken cancellation)
+        {
+            AppDataRequest adr = new(userId, recordKey);
+
+            //If cache is disabled, or the NoCache flag is set, bypass the cache
+            if (_cache is null || (flags & RecordOpFlags.NoCache) > 0)
+            {
+                return _backingStore.GetAsync(adr, cancellation);
+            }
+
+            return _cache.FetchAsync(
+                request: adr,
+                resultFactory: _backingStore.GetAsync, 
+                cancellation
+            );
+        }
+
+        ///<inheritdoc/>
+        public Task SetRecordAsync(string userId, string recordKey, byte[] data, ulong checksum, RecordOpFlags flags, CancellationToken cancellation)
+        {
+            AppDataRequest adr = new (userId, recordKey);
+
+            UserRecordData record = new ()
+            {
+                Data            = data,
+                Checksum        = checksum,
+
+                /*
+                 * Cache upsert should set this to the current time. Set to 0 to force expire by default
+                 * The database does not map this value so it doesn't matter for the backing store
+                 */
+                CacheTimestamp  = 0
+            };
+
+            //If cache is disabled, or the NoCache flag is set, bypass the cache
+            if (_cache is null || (flags & RecordOpFlags.NoCache) > 0)
+            {
+                return _backingStore.UpsertAsync(adr, record, cancellation);
+            }
+
+            return _cache.UpsertAsync(
+                request: adr,
+                entity: record,
+                action: _backingStore.UpsertAsync,
+                cancellation
+            );
+        }
+
+        private void OnTaskError(Task update)
+        {
+            try
+            {
+                update.GetAwaiter().GetResult();
+            }
+            catch (Exception e)
+            {
+                if (_logger.IsEnabled(LogLevel.Debug))
+                {
+                    _logger.Warn(e, "Failed to update cached User AppData record");
+                }
+                else
+                {
+                    _logger.Warn("Failed to update cached AppData record");
+                }
+            }
+        }
+
+        private sealed class MpSerializer : ICacheObjectDeserializer, ICacheObjectSerializer
+        {
+
+            ///<inheritdoc/>
+            public T? Deserialize<T>(ReadOnlySpan<byte> objectData) 
+                => MemoryPackSerializer.Deserialize<T>(objectData);
+
+            ///<inheritdoc/>
+            public void Serialize<T>(T obj, IBufferWriter<byte> finiteWriter)
+                => MemoryPackSerializer.Serialize(finiteWriter, obj);
+        }
+
+        private sealed class MpCachePolicy(TimeSpan CacheTTL) : ICacheExpirationPolicy<UserRecordData>
+        {
+            ///<inheritdoc/>
+            public bool IsExpired(UserRecordData result)
+            {
+                DateTimeOffset timestamp = DateTimeOffset.FromUnixTimeSeconds(result.CacheTimestamp);
+
+                return timestamp.Add(CacheTTL) > DateTimeOffset.UtcNow;
+            }
+
+            ///<inheritdoc/>
+            public void OnRefreshed(UserRecordData entity)
+            {
+                //Store current utc timestamp on entity before its stored in cache again
+                entity.CacheTimestamp = DateTimeOffset.UtcNow.ToUnixTimeSeconds();
+            }
+        }
+
+        sealed class CacheConfig
+        {
+            [JsonPropertyName("enabled")]
+            public bool Enabled { get; set; } = true;
+
+            [JsonPropertyName("ttl")]
+            public long CacheTTL { get; set; } = 120;    //max age in seconds
+
+            [JsonPropertyName("force_write_back")]
+            public bool WriteBack { get; set; } = false;
+
+            [JsonPropertyName("prefix")]
+            public string? Prefix { get; set; }
+        }
+    }
+}