Feat/usage stats collection (#317)

* feat: implement anonymous usage data collection Co-authored-by: Joseph Milazzo <joseph.v.milazzo@gmail.com>
2021-06-20 19:26:35 -03:00 · 2021-06-20 19:26:35 -03:00 · 1c9b2572ae
commit 1c9b2572ae
parent b25335acbd
23 changed files with 613 additions and 17 deletions
--- a/API/Services/Clients/StatsApiClient.cs
+++ b/API/Services/Clients/StatsApiClient.cs
@ -0,0 +1,55 @@
+using System;
+using System.Net.Http;
+using System.Net.Http.Json;
+using System.Threading.Tasks;
+using API.DTOs;
+using Microsoft.Extensions.Logging;
+
+namespace API.Services.Clients
+{
+    public class StatsApiClient
+    {
+        private readonly HttpClient _client;
+        private readonly ILogger<StatsApiClient> _logger;
+
+        public StatsApiClient(HttpClient client, ILogger<StatsApiClient> logger)
+        {
+            _client = client;
+            _logger = logger;
+        }
+
+        public async Task SendDataToStatsServer(UsageStatisticsDto data)
+        {
+            var responseContent = string.Empty;
+
+            try
+            {
+                var response = await _client.PostAsJsonAsync("/api/InstallationStats", data);
+
+                responseContent = await response.Content.ReadAsStringAsync();
+
+                response.EnsureSuccessStatusCode();
+            }
+            catch (HttpRequestException e)
+            {
+                var info = new
+                {
+                    dataSent = data,
+                    response = responseContent
+                };
+
+                _logger.LogError(e, "The StatsServer did not respond successfully. {Content}", info);
+
+                Console.WriteLine(e);
+                throw;
+            }
+            catch (Exception e)
+            {
+                _logger.LogError(e, "An error happened during the request to the Stats Server");
+
+                Console.WriteLine(e);
+                throw;
+            }
+        }
+    }
+}
--- a/API/Services/HostedServices/StartupTasksHostedService.cs
+++ b/API/Services/HostedServices/StartupTasksHostedService.cs
@ -0,0 +1,54 @@
+using System;
+using System.Threading;
+using System.Threading.Tasks;
+using API.Interfaces;
+using API.Interfaces.Services;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Hosting;
+
+namespace API.Services.HostedServices
+{
+    public class StartupTasksHostedService : IHostedService
+    {
+        private readonly IServiceProvider _provider;
+
+        public StartupTasksHostedService(IServiceProvider serviceProvider)
+        {
+            _provider = serviceProvider;
+        }
+
+        public async Task StartAsync(CancellationToken cancellationToken)
+        {
+            using var scope = _provider.CreateScope();
+
+            var taskScheduler = scope.ServiceProvider.GetRequiredService<ITaskScheduler>();
+            taskScheduler.ScheduleTasks();
+
+            try
+            {
+                await ManageStartupStatsTasks(scope, taskScheduler);
+            }
+            catch (Exception e)
+            {
+                //If stats startup fail the user can keep using the app
+            }
+        }
+
+        private async Task ManageStartupStatsTasks(IServiceScope serviceScope, ITaskScheduler taskScheduler)
+        {
+            var settingsRepository = serviceScope.ServiceProvider.GetRequiredService<ISettingsRepository>();
+
+            var settingsDto = await settingsRepository.GetSettingsDtoAsync();
+
+            if (!settingsDto.AllowStatCollection) return;
+
+            taskScheduler.ScheduleStatsTasks();
+
+            var statsService = serviceScope.ServiceProvider.GetRequiredService<IStatsService>();
+
+            await statsService.CollectAndSendStatsData();
+        }
+
+        public Task StopAsync(CancellationToken cancellationToken) => Task.CompletedTask;
+    }
+}
--- a/API/Services/StatsService.cs
+++ b/API/Services/StatsService.cs
@ -0,0 +1,186 @@
+using System;
+using System.IO;
+using System.Linq;
+using System.Runtime.InteropServices;
+using System.Text.Json;
+using System.Threading;
+using System.Threading.Tasks;
+using API.Data;
+using API.DTOs;
+using API.Interfaces;
+using API.Interfaces.Services;
+using API.Services.Clients;
+using Kavita.Common;
+using Kavita.Common.EnvironmentInfo;
+using Microsoft.EntityFrameworkCore;
+using Microsoft.Extensions.Logging;
+
+namespace API.Services
+{
+    public class StatsService : IStatsService
+    {
+        private const string TempFilePath = "stats/";
+        private const string TempFileName = "app_stats.json";
+
+        private readonly StatsApiClient _client;
+        private readonly DataContext _dbContext;
+        private readonly ILogger<StatsService> _logger;
+        private readonly IFileRepository _fileRepository;
+
+        public StatsService(StatsApiClient client, DataContext dbContext, ILogger<StatsService> logger,
+            IFileRepository fileRepository)
+        {
+            _client = client;
+            _dbContext = dbContext;
+            _logger = logger;
+            _fileRepository = fileRepository;
+        }
+
+        private static string FinalPath => Path.Combine(Directory.GetCurrentDirectory(), TempFilePath, TempFileName);
+        private static bool FileExists => File.Exists(FinalPath);
+
+        public async Task PathData(ClientInfoDto clientInfoDto)
+        {
+            _logger.LogInformation("Pathing client data to the file");
+
+            var statisticsDto = await GetData();
+
+            statisticsDto.AddClientInfo(clientInfoDto);
+
+            await SaveFile(statisticsDto);
+        }
+
+        public async Task CollectRelevantData()
+        {
+            _logger.LogInformation("Collecting data from the server and database");
+
+            _logger.LogInformation("Collecting usage info");
+            var usageInfo = await GetUsageInfo();
+
+            _logger.LogInformation("Collecting server info");
+            var serverInfo = GetServerInfo();
+
+            await PathData(serverInfo, usageInfo);
+        }
+
+        public async Task FinalizeStats()
+        {
+            try
+            {
+                _logger.LogInformation("Finalizing Stats collection flow");
+
+                var data = await GetExistingData<UsageStatisticsDto>();
+
+                _logger.LogInformation("Sending data to the Stats server");
+                await _client.SendDataToStatsServer(data);
+
+                _logger.LogInformation("Deleting the file from disk");
+                if (FileExists) File.Delete(FinalPath);
+            }
+            catch (Exception e)
+            {
+                _logger.LogError("Error Finalizing Stats collection flow", e);
+                throw;
+            }
+        }
+
+        public async Task CollectAndSendStatsData()
+        {
+            await CollectRelevantData();
+            await FinalizeStats();
+        }
+
+        private async Task PathData(ServerInfoDto serverInfoDto, UsageInfoDto usageInfoDto)
+        {
+            _logger.LogInformation("Pathing server and usage info to the file");
+
+            var data = await GetData();
+
+            data.ServerInfo = serverInfoDto;
+            data.UsageInfo = usageInfoDto;
+
+            data.MarkAsUpdatedNow();
+
+            await SaveFile(data);
+        }
+
+        private async ValueTask<UsageStatisticsDto> GetData()
+        {
+            if (!FileExists) return new UsageStatisticsDto {InstallId = HashUtil.AnonymousToken()};
+
+            return await GetExistingData<UsageStatisticsDto>();
+        }
+
+        private async Task<UsageInfoDto> GetUsageInfo()
+        {
+            var usersCount = await _dbContext.Users.CountAsync();
+
+            var libsCountByType = await _dbContext.Library
+                .AsNoTracking()
+                .GroupBy(x => x.Type)
+                .Select(x => new LibInfo {Type = x.Key, Count = x.Count()})
+                .ToArrayAsync();
+
+            var uniqueFileTypes = await _fileRepository.GetFileExtensions();
+
+            var usageInfo = new UsageInfoDto
+            {
+                UsersCount = usersCount,
+                LibraryTypesCreated = libsCountByType,
+                FileTypes = uniqueFileTypes
+            };
+
+            return usageInfo;
+        }
+
+        private static ServerInfoDto GetServerInfo()
+        {
+            var serverInfo = new ServerInfoDto
+            {
+                Os = RuntimeInformation.OSDescription,
+                DotNetVersion = Environment.Version.ToString(),
+                RunTimeVersion = RuntimeInformation.FrameworkDescription,
+                KavitaVersion = BuildInfo.Version.ToString(),
+                Culture = Thread.CurrentThread.CurrentCulture.Name,
+                BuildBranch = BuildInfo.Branch
+            };
+
+            return serverInfo;
+        }
+
+        private async Task<T> GetExistingData<T>()
+        {
+            _logger.LogInformation("Fetching existing data from file");
+            var existingDataJson = await GetFileDataAsString();
+
+            _logger.LogInformation("Deserializing data from file to object");
+            var existingData = JsonSerializer.Deserialize<T>(existingDataJson);
+
+            return existingData;
+        }
+
+        private async Task<string> GetFileDataAsString()
+        {
+            _logger.LogInformation("Reading file from disk");
+            return await File.ReadAllTextAsync(FinalPath);
+        }
+
+        private async Task SaveFile(UsageStatisticsDto statisticsDto)
+        {
+            _logger.LogInformation("Saving file");
+
+            var finalDirectory = FinalPath.Replace(TempFileName, string.Empty);
+            if (!Directory.Exists(finalDirectory))
+            {
+                _logger.LogInformation("Creating tmp directory");
+                Directory.CreateDirectory(finalDirectory);
+            }
+
+            _logger.LogInformation("Serializing data to write");
+            var dataJson = JsonSerializer.Serialize(statisticsDto);
+
+            _logger.LogInformation("Writing file to the disk");
+            await File.WriteAllTextAsync(FinalPath, dataJson);
+        }
+    }
+}
--- a/API/Services/TaskScheduler.cs
+++ b/API/Services/TaskScheduler.cs
@ -19,11 +19,14 @@ namespace API.Services
        private readonly IBackupService _backupService;
        private readonly ICleanupService _cleanupService;

+        private readonly IStatsService _statsService;
+
        public static BackgroundJobServer Client => new BackgroundJobServer();


        public TaskScheduler(ICacheService cacheService, ILogger<TaskScheduler> logger, IScannerService scannerService, 
-            IUnitOfWork unitOfWork, IMetadataService metadataService, IBackupService backupService, ICleanupService cleanupService)
+            IUnitOfWork unitOfWork, IMetadataService metadataService, IBackupService backupService,
+            ICleanupService cleanupService, IStatsService statsService)
        {
            _cacheService = cacheService;
            _logger = logger;
@ -32,6 +35,7 @@ namespace API.Services
            _metadataService = metadataService;
            _backupService = backupService;
            _cleanupService = cleanupService;
+            _statsService = statsService;
        }

        public void ScheduleTasks()
@ -65,6 +69,33 @@ namespace API.Services
            RecurringJob.AddOrUpdate("cleanup", () => _cleanupService.Cleanup(), Cron.Daily);
        }

+        #region StatsTasks
+
+        private const string SendDataTask = "finalize-stats";
+        public void ScheduleStatsTasks()
+        {
+            var allowStatCollection = bool.Parse(Task.Run(() => _unitOfWork.SettingsRepository.GetSettingAsync(ServerSettingKey.AllowStatCollection)).GetAwaiter().GetResult().Value);
+            if (!allowStatCollection)
+            {
+                _logger.LogDebug("User has opted out of stat collection, not registering tasks");
+                return;
+            }
+            
+            _logger.LogDebug("Adding StatsTasks");
+
+            _logger.LogDebug("Scheduling Send data to the Stats server {Setting}", nameof(Cron.Daily));
+            RecurringJob.AddOrUpdate(SendDataTask, () => _statsService.CollectAndSendStatsData(), Cron.Daily);
+        }
+
+        public void CancelStatsTasks()
+        {
+            _logger.LogDebug("Cancelling/Removing StatsTasks");
+
+            RecurringJob.RemoveIfExists(SendDataTask);
+        }
+
+        #endregion
+
        public void ScanLibrary(int libraryId, bool forceUpdate = false)
        {
            _logger.LogInformation("Enqueuing library scan for: {LibraryId}", libraryId);