AZEN-SGG/Kavita
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Stats Rework (#765)

Browse source 
* Fixed a duplicate check for updates. Changed checking from weekly to daily.

* Refactored how dark variables were accessed to reduce size of component css. Refactored Stats code to use lesser information for reporting.

* Use the installId from the database which is most unlikely to change.

* Fixed a missing interface with stat service

* Added DotnetVersion back into collection

* Updated url to new host.
This commit is contained in:
Joseph Milazzo 2021-11-16 15:11:17 -06:00 • committed by GitHub
parent b2831c7606
commit 1ada34984f
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
17 changed files with 44 additions and 349 deletions
Download patch file Download diff file Expand all files Collapse all files

8
API/Controllers/ServerController.cs
View file

@ -26,10 +26,11 @@ namespace API.Controllers
private readonly IArchiveService _archiveService;
private readonly ICacheService _cacheService;
private readonly IVersionUpdaterService _versionUpdaterService;
private readonly IStatsService _statsService;
public ServerController(IHostApplicationLifetime applicationLifetime, ILogger<ServerController> logger, IConfiguration config,
IBackupService backupService, IArchiveService archiveService, ICacheService cacheService,
IVersionUpdaterService versionUpdaterService)
IVersionUpdaterService versionUpdaterService, IStatsService statsService)
{
_applicationLifetime = applicationLifetime;
_logger = logger;
@ -38,6 +39,7 @@ namespace API.Controllers
_archiveService = archiveService;
_cacheService = cacheService;
_versionUpdaterService = versionUpdaterService;
_statsService = statsService;
}
/// <summary>
@ -84,9 +86,9 @@ namespace API.Controllers
/// </summary>
/// <returns></returns>
[HttpGet("server-info")]
public ActionResult<ServerInfoDto> GetVersion()
public async Task<ActionResult<ServerInfoDto>> GetVersion()
{
return Ok(StatsService.GetServerInfo());
return Ok(await _statsService.GetServerInfo());
}
[HttpGet("logs")]

39
API/Controllers/StatsController.cs
View file

@ -1,39 +0,0 @@
using System;
using System.Threading.Tasks;
using API.DTOs.Stats;
using API.Interfaces.Services;
using Microsoft.AspNetCore.Authorization;
using Microsoft.AspNetCore.Mvc;
using Microsoft.Extensions.Logging;
namespace API.Controllers
{
public class StatsController : BaseApiController
{
private readonly ILogger<StatsController> _logger;
private readonly IStatsService _statsService;
public StatsController(ILogger<StatsController> logger, IStatsService statsService)
{
_logger = logger;
_statsService = statsService;
}
[AllowAnonymous]
[HttpPost("client-info")]
public async Task<IActionResult> AddClientInfo([FromBody] ClientInfoDto clientInfoDto)
{
try
{
await _statsService.RecordClientInfo(clientInfoDto);
return Ok();
}
catch (Exception ex)
{
_logger.LogError(ex, "Error updating the usage statistics");
throw;
}
}
}
}

37
API/DTOs/Stats/ClientInfoDto.cs
View file

@ -1,37 +0,0 @@
using System;
namespace API.DTOs.Stats
{
public class ClientInfoDto
{
public ClientInfoDto()
{
CollectedAt = DateTime.UtcNow;
}
public string KavitaUiVersion { get; set; }
public string ScreenResolution { get; set; }
public string PlatformType { get; set; }
public DetailsVersion Browser { get; set; }
public DetailsVersion Os { get; set; }
public DateTime? CollectedAt { get; set; }
public bool UsingDarkTheme { get; set; }
public bool IsTheSameDevice(ClientInfoDto clientInfoDto)
{
return (clientInfoDto.ScreenResolution ?? string.Empty).Equals(ScreenResolution) &&
(clientInfoDto.PlatformType ?? string.Empty).Equals(PlatformType) &&
(clientInfoDto.Browser?.Name ?? string.Empty).Equals(Browser?.Name) &&
(clientInfoDto.Os?.Name ?? string.Empty).Equals(Os?.Name) &&
clientInfoDto.CollectedAt.GetValueOrDefault().ToString("yyyy-MM-dd")
.Equals(CollectedAt.GetValueOrDefault().ToString("yyyy-MM-dd"));
}
}
public class DetailsVersion
{
public string Name { get; set; }
public string Version { get; set; }
}
}

8
API/DTOs/Stats/ServerInfoDto.cs
View file

@ -2,13 +2,11 @@
{
public class ServerInfoDto
{
public string InstallId { get; set; }
public string Os { get; set; }
public string DotNetVersion { get; set; }
public string RunTimeVersion { get; set; }
public string KavitaVersion { get; set; }
public string BuildBranch { get; set; }
public string Culture { get; set; }
public bool IsDocker { get; set; }
public string DotnetVersion { get; set; }
public string KavitaVersion { get; set; }
public int NumOfCores { get; set; }
}
}

24
API/DTOs/Stats/UsageInfoDto.cs
View file

@ -1,24 +0,0 @@
using System.Collections.Generic;
using API.Entities.Enums;
namespace API.DTOs.Stats
{
public class UsageInfoDto
{
public UsageInfoDto()
{
FileTypes = new HashSet<string>();
LibraryTypesCreated = new HashSet<LibInfo>();
}
public int UsersCount { get; set; }
public IEnumerable<string> FileTypes { get; set; }
public IEnumerable<LibInfo> LibraryTypesCreated { get; set; }
}
public class LibInfo
{
public LibraryType Type { get; set; }
public int Count { get; set; }
}
}

33
API/DTOs/Stats/UsageStatisticsDto.cs
View file

@ -1,33 +0,0 @@
using System;
using System.Collections.Generic;
using System.Linq;
namespace API.DTOs.Stats
{
public class UsageStatisticsDto
{
public UsageStatisticsDto()
{
MarkAsUpdatedNow();
ClientsInfo = new List<ClientInfoDto>();
}
public string InstallId { get; set; }
public DateTime LastUpdate { get; set; }
public UsageInfoDto UsageInfo { get; set; }
public ServerInfoDto ServerInfo { get; set; }
public List<ClientInfoDto> ClientsInfo { get; set; }
public void MarkAsUpdatedNow()
{
LastUpdate = DateTime.UtcNow;
}
public void AddClientInfo(ClientInfoDto clientInfoDto)
{
if (ClientsInfo.Any(x => x.IsTheSameDevice(clientInfoDto))) return;
ClientsInfo.Add(clientInfoDto);
}
}
}

2
API/Interfaces/Services/IStatsService.cs
View file

@ -5,7 +5,7 @@ namespace API.Interfaces.Services
{
public interface IStatsService
{
Task RecordClientInfo(ClientInfoDto clientInfoDto);
Task Send();
Task<ServerInfoDto> GetServerInfo();
}
}

1
API/Services/DirectoryService.cs
View file

@ -21,7 +21,6 @@ namespace API.Services
public static readonly string CacheDirectory = Path.Join(Directory.GetCurrentDirectory(), "config", "cache");
public static readonly string CoverImageDirectory = Path.Join(Directory.GetCurrentDirectory(), "config", "covers");
public static readonly string BackupDirectory = Path.Join(Directory.GetCurrentDirectory(), "config", "backups");
public static readonly string StatsDirectory = Path.Join(Directory.GetCurrentDirectory(), "config", "stats");
public static readonly string ConfigDirectory = Path.Join(Directory.GetCurrentDirectory(), "config");
public DirectoryService(ILogger<DirectoryService> logger)

8
API/Services/TaskScheduler.cs
View file

@ -74,8 +74,6 @@ namespace API.Services
RecurringJob.AddOrUpdate("cleanup", () => _cleanupService.Cleanup(), Cron.Daily, TimeZoneInfo.Local);
// Schedule update check between noon and 6pm local time
RecurringJob.AddOrUpdate("check-for-updates", () => _scannerService.ScanLibraries(), Cron.Daily(Rnd.Next(12, 18)), TimeZoneInfo.Local);
}
#region StatsTasks
@ -91,7 +89,7 @@ namespace API.Services
}
_logger.LogDebug("Scheduling stat collection daily");
RecurringJob.AddOrUpdate(SendDataTask, () => _statsService.Send(), Cron.Daily, TimeZoneInfo.Local);
RecurringJob.AddOrUpdate(SendDataTask, () => _statsService.Send(), Cron.Daily(Rnd.Next(0, 22)), TimeZoneInfo.Local);
}
public void CancelStatsTasks()
@ -114,8 +112,8 @@ namespace API.Services
public void ScheduleUpdaterTasks()
{
_logger.LogInformation("Scheduling Auto-Update tasks");
RecurringJob.AddOrUpdate("check-updates", () => CheckForUpdate(), Cron.Weekly, TimeZoneInfo.Local);
// Schedule update check between noon and 6pm local time
RecurringJob.AddOrUpdate("check-updates", () => _versionUpdaterService.CheckForUpdate(), Cron.Daily(Rnd.Next(12, 18)), TimeZoneInfo.Local);
}
#endregion

150
API/Services/Tasks/StatsService.cs
View file

@ -1,44 +1,29 @@
using System;
using System.IO;
using System.Linq;
using System.Net.Http;
using System.Runtime.InteropServices;
using System.Text.Json;
using System.Threading;
using System.Threading.Tasks;
using API.Data;
using API.DTOs.Stats;
using API.Entities.Enums;
using API.Interfaces;
using API.Interfaces.Services;
using Flurl.Http;
using Hangfire;
using Kavita.Common;
using Kavita.Common.EnvironmentInfo;
using Microsoft.AspNetCore.Http;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Logging;
namespace API.Services.Tasks
{
public class StatsService : IStatsService
{
private const string StatFileName = "app_stats.json";
private readonly DataContext _dbContext;
private readonly ILogger<StatsService> _logger;
private readonly IUnitOfWork _unitOfWork;
#pragma warning disable S1075
private const string ApiUrl = "http://stats.kavitareader.com";
private const string ApiUrl = "http://stats2.kavitareader.com";
#pragma warning restore S1075
private static readonly string StatsFilePath = Path.Combine(DirectoryService.StatsDirectory, StatFileName);
private static bool FileExists => File.Exists(StatsFilePath);
public StatsService(DataContext dbContext, ILogger<StatsService> logger,
IUnitOfWork unitOfWork)
public StatsService(ILogger<StatsService> logger, IUnitOfWork unitOfWork)
{
_dbContext = dbContext;
_logger = logger;
_unitOfWork = unitOfWork;
}
@ -55,17 +40,7 @@ namespace API.Services.Tasks
return;
}
var rnd = new Random();
var offset = rnd.Next(0, 6);
if (offset == 0)
{
await SendData();
}
else
{
_logger.LogInformation("KavitaStats upload has been schedule to run in {Offset} hours", offset);
BackgroundJob.Schedule(() => SendData(), DateTimeOffset.Now.AddHours(offset));
}
await SendData();
}
/// <summary>
@ -74,55 +49,21 @@ namespace API.Services.Tasks
// ReSharper disable once MemberCanBePrivate.Global
public async Task SendData()
{
await CollectRelevantData();
await FinalizeStats();
var data = await GetServerInfo();
await SendDataToStatsServer(data);
}
public async Task RecordClientInfo(ClientInfoDto clientInfoDto)
{
var statisticsDto = await GetData();
statisticsDto.AddClientInfo(clientInfoDto);
await SaveFile(statisticsDto);
}
private async Task CollectRelevantData()
{
var usageInfo = await GetUsageInfo();
var serverInfo = GetServerInfo();
await PathData(serverInfo, usageInfo);
}
private async Task FinalizeStats()
{
try
{
var data = await GetExistingData<UsageStatisticsDto>();
var successful = await SendDataToStatsServer(data);
if (successful)
{
ResetStats();
}
}
catch (Exception ex)
{
_logger.LogError(ex, "There was an exception while sending data to KavitaStats");
}
}
private async Task<bool> SendDataToStatsServer(UsageStatisticsDto data)
private async Task SendDataToStatsServer(ServerInfoDto data)
{
var responseContent = string.Empty;
try
{
var response = await (ApiUrl + "/api/InstallationStats")
var response = await (ApiUrl + "/api/v2/stats")
.WithHeader("Accept", "application/json")
.WithHeader("User-Agent", "Kavita")
.WithHeader("x-api-key", "MsnvA2DfQqxSK5jh")
.WithHeader("api-key", "MsnvA2DfQqxSK5jh")
.WithHeader("x-kavita-version", BuildInfo.Version)
.WithTimeout(TimeSpan.FromSeconds(30))
.PostJsonAsync(data);
@ -130,10 +71,7 @@ namespace API.Services.Tasks
if (response.StatusCode != StatusCodes.Status200OK)
{
_logger.LogError("KavitaStats did not respond successfully. {Content}", response);
return false;
}
return true;
}
catch (HttpRequestException e)
{
@ -149,84 +87,22 @@ namespace API.Services.Tasks
{
_logger.LogError(e, "An error happened during the request to KavitaStats");
}
return false;
}
private static void ResetStats()
{
if (FileExists) File.Delete(StatsFilePath);
}
private async Task PathData(ServerInfoDto serverInfoDto, UsageInfoDto usageInfoDto)
{
var data = await GetData();
data.ServerInfo = serverInfoDto;
data.UsageInfo = usageInfoDto;
data.MarkAsUpdatedNow();
await SaveFile(data);
}
private static async ValueTask<UsageStatisticsDto> GetData()
{
if (!FileExists) return new UsageStatisticsDto {InstallId = HashUtil.AnonymousToken()};
return await GetExistingData<UsageStatisticsDto>();
}
private async Task<UsageInfoDto> GetUsageInfo()
{
var usersCount = await _dbContext.Users.CountAsync();
var libsCountByType = await _dbContext.Library
.AsNoTracking()
.GroupBy(x => x.Type)
.Select(x => new LibInfo {Type = x.Key, Count = x.Count()})
.ToArrayAsync();
var uniqueFileTypes = await _unitOfWork.FileRepository.GetFileExtensions();
var usageInfo = new UsageInfoDto
{
UsersCount = usersCount,
LibraryTypesCreated = libsCountByType,
FileTypes = uniqueFileTypes
};
return usageInfo;
}
public static ServerInfoDto GetServerInfo()
public async Task<ServerInfoDto> GetServerInfo()
{
var installId = await _unitOfWork.SettingsRepository.GetSettingAsync(ServerSettingKey.InstallId);
var serverInfo = new ServerInfoDto
{
InstallId = installId.Value,
Os = RuntimeInformation.OSDescription,
DotNetVersion = Environment.Version.ToString(),
RunTimeVersion = RuntimeInformation.FrameworkDescription,
KavitaVersion = BuildInfo.Version.ToString(),
Culture = Thread.CurrentThread.CurrentCulture.Name,
BuildBranch = BuildInfo.Branch,
DotnetVersion = Environment.Version.ToString(),
IsDocker = new OsInfo(Array.Empty<IOsVersionAdapter>()).IsDocker,
NumOfCores = Environment.ProcessorCount
NumOfCores = Math.Max(Environment.ProcessorCount, 1)
};
return serverInfo;
}
private static async Task<T> GetExistingData<T>()
{
var json = await File.ReadAllTextAsync(StatsFilePath);
return JsonSerializer.Deserialize<T>(json);
}
private static async Task SaveFile(UsageStatisticsDto statisticsDto)
{
DirectoryService.ExistOrCreate(DirectoryService.StatsDirectory);
await File.WriteAllTextAsync(StatsFilePath, JsonSerializer.Serialize(statisticsDto));
}
}
}