Skip to content

Commit

Permalink
Implement retry for failed files in Slskd
Browse files Browse the repository at this point in the history
  • Loading branch information
Meyn committed Jan 22, 2025
1 parent 11ee76f commit 4977287
Show file tree
Hide file tree
Showing 11 changed files with 177 additions and 54 deletions.
5 changes: 4 additions & 1 deletion README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,10 @@ Tubifarry supports **Slskd**, the Soulseek client, as both an **indexer** and **

---

### YouTube Downloader Setup 🎥
### YouTube Downloader Setup 🎥
> #### YouTube Warning ⚠️
> Please be aware that YouTube often blocks Tubifarry as a bot. We are currently waiting for external updates. Logging in and the YouTube-only indexer are disabled for now. If login is necessary, please revert to versions earlier than 1.6.0. We appreciate your patience and understanding during this time.

Tubifarry allows you to download music directly from YouTube. Follow the steps below to configure the YouTube downloader.

#### **Configure the Indexer**:
Expand Down
16 changes: 11 additions & 5 deletions Tubifarry/Core/Model/AlbumData.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,8 @@ namespace Tubifarry.Core.Model
/// </summary>
public class AlbumData
{
public string? Guid { get; set; }

public string IndexerName { get; }
// Mixed
public string AlbumId { get; set; } = string.Empty;
Expand All @@ -34,6 +36,7 @@ public class AlbumData
// Soulseek
public long? Size { get; set; }
public int Priotity { get; set; }
public string? ExtraInfo { get; set; }

// Not used
public AudioFormat Codec { get; set; } = AudioFormat.AAC;
Expand All @@ -45,7 +48,7 @@ public class AlbumData
/// </summary>
public ReleaseInfo ToReleaseInfo() => new()
{
Guid = $"{IndexerName}-{AlbumId}-{Bitrate}",
Guid = Guid ?? $"{IndexerName}-{AlbumId}-{Bitrate}",
Artist = ArtistName,
Album = AlbumName,
DownloadUrl = AlbumId,
Expand Down Expand Up @@ -92,12 +95,15 @@ private string ConstructTitle()
calculatedBitrate = (int)(Size.Value * 8 / (Duration * 1000));

if (AudioFormatHelper.IsLossyFormat(Codec) && calculatedBitrate != 0)
title += $" [{Codec} {calculatedBitrate}kbps] [WEB]";
if (!AudioFormatHelper.IsLossyFormat(Codec) && BitDepth != 0)
title += $" [{Codec} {BitDepth}bit] [WEB]";
title += $" [{Codec} {calculatedBitrate}kbps]";
else if (!AudioFormatHelper.IsLossyFormat(Codec) && BitDepth != 0)
title += $" [{Codec} {BitDepth}bit]";
else
title += $" [{Codec}] [WEB]";
title += $" [{Codec}]";
if (ExtraInfo != null)
title += $" [{ExtraInfo}]";

title += " [WEB]";
return title;
}

Expand Down
74 changes: 62 additions & 12 deletions Tubifarry/Download/Clients/Soulseek/SlskdClient.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,26 +32,71 @@ public SlskdClient(IHttpClient httpClient, IConfigService configService, IDiskPr
public override async Task<string> Download(RemoteAlbum remoteAlbum, IIndexer indexer)
{
SlskdDownloadItem item = new(remoteAlbum);
HttpRequest request = BuildHttpRequest(remoteAlbum.Release.DownloadUrl, HttpMethod.Post, remoteAlbum.Release.Source);
HttpResponse response = await _httpClient.ExecuteAsync(request);
try
{
HttpRequest request = BuildHttpRequest(remoteAlbum.Release.DownloadUrl, HttpMethod.Post, remoteAlbum.Release.Source);
HttpResponse response = await _httpClient.ExecuteAsync(request);

if (response.StatusCode != HttpStatusCode.Created)
throw new DownloadClientException("Failed to create download.");
if (Settings.UseLRCLIB)
if (response.StatusCode != HttpStatusCode.Created)
throw new DownloadClientException("Failed to create download.");
item.FileStateChanged += FileStateChanged;
AddDownloadItem(item);
AddDownloadItem(item);
}
catch (Exception)
{
try
{
RemoveItemAsync(item).Wait();
}
catch (Exception) { }
return null!;
}
return item.ID.ToString();
}

private void FileStateChanged(object? sender, SlskdDownloadFile file)
private void FileStateChanged(object? sender, SlskdFileState fileState)
{
string filename = file.Filename;
fileState.UpdateMaxRetryCount(Settings.RetryAttempts);
string filename = fileState.File.Filename;
string extension = Path.GetExtension(filename);
AudioFormat format = AudioFormatHelper.GetAudioCodecFromExtension(extension.TrimStart('.'));
if (fileState.GetStatus() == DownloadItemStatus.Warning)
{
_logger.Trace($"Retrying download for file: {filename}. Attempt {fileState.RetryCount} of {fileState.MaxRetryCount}");
_ = RetryDownloadAsync(fileState, (SlskdDownloadItem)sender!);
return;
}

if (file.GetStatus() != DownloadItemStatus.Completed || format == AudioFormat.Unknown)
if (fileState.GetStatus() != DownloadItemStatus.Completed || format == AudioFormat.Unknown)
return;
PostProcess((SlskdDownloadItem)sender!, file);
if (Settings.UseLRCLIB)
PostProcess((SlskdDownloadItem)sender!, fileState.File);
}

private async Task RetryDownloadAsync(SlskdFileState fileState, SlskdDownloadItem item)
{
try
{
using JsonDocument doc = JsonDocument.Parse(item.RemoteAlbum.Release.Source);
JsonElement root = doc.RootElement;
JsonElement matchingItem = root.EnumerateArray()
.FirstOrDefault(x => x.GetProperty("Filename").GetString() == fileState.File.Filename);

if (matchingItem.ValueKind == JsonValueKind.Undefined)
return;
string payload = JsonSerializer.Serialize(new[] { matchingItem });

HttpRequest request = BuildHttpRequest(item.RemoteAlbum.Release.DownloadUrl, HttpMethod.Post, payload);
HttpResponse response = await _httpClient.ExecuteAsync(request);

if (response.StatusCode == HttpStatusCode.Created)
_logger.Trace($"Successfully retried download for file: {fileState.File.Filename}");
}
catch (Exception ex)
{
_logger.Error(ex, $"Failed to retry download for file: {fileState.File.Filename}");
}
fileState.IncrementAttempt();
}

private void PostProcess(SlskdDownloadItem item, SlskdDownloadFile file) => item.PostProcessTasks.Add(Task.Run(async () =>
Expand Down Expand Up @@ -111,8 +156,13 @@ private async Task UpdateDownloadItemsAsync()
foreach (SlskdDownloadDirectory dir in data)
{
HashCode hash = new();
foreach (SlskdDownloadFile file in dir.Files ?? new List<SlskdDownloadFile>())
hash.Add(file.Filename);
List<string> sortedFilenames = dir.Files?
.Select(file => file.Filename)
.OrderBy(filename => filename)
.ToList() ?? new List<string>();

foreach (string? filename in sortedFilenames)
hash.Add(filename);
SlskdDownloadItem? item = GetDownloadItem(hash.ToHashCode());
if (item == null)
continue;
Expand Down
114 changes: 84 additions & 30 deletions Tubifarry/Download/Clients/Soulseek/SlskdModels.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,12 +19,12 @@ public class SlskdDownloadItem
public string? Username { get; set; }
public RemoteAlbum RemoteAlbum { get; set; }

public event EventHandler<SlskdDownloadFile>? FileStateChanged;
public event EventHandler<SlskdFileState>? FileStateChanged;

private Logger _logger;

private SlskdDownloadDirectory? _slskdDownloadDirectory;
private Dictionary<string, string> _previousFileStates = new();
private Dictionary<string, SlskdFileState> _previousFileStates = new();

public List<Task> PostProcessTasks { get; } = new();

Expand All @@ -35,7 +35,7 @@ public SlskdDownloadDirectory? SlskdDownloadDirectory
{
if (_slskdDownloadDirectory == value)
return;
CompareFileStates(_slskdDownloadDirectory, value);
CompareFileStates(value);
_slskdDownloadDirectory = value;
}
}
Expand All @@ -48,21 +48,34 @@ public SlskdDownloadItem(RemoteAlbum remoteAlbum)
_lastUpdateTime = DateTime.UtcNow;
_lastDownloadedSize = 0;
HashCode hash = new();
foreach (SlskdFileData file in FileData)
hash.Add(file.Filename);
List<string?> sortedFilenames = FileData
.Select(file => file.Filename)
.OrderBy(filename => filename)
.ToList();
foreach (string? filename in sortedFilenames)
hash.Add(filename);
ID = hash.ToHashCode();
_downloadClientItem = new() { DownloadId = ID.ToString(), CanBeRemoved = true, CanMoveFiles = true };
}

private void CompareFileStates(SlskdDownloadDirectory? previousDirectory, SlskdDownloadDirectory? newDirectory)
private void CompareFileStates(SlskdDownloadDirectory? newDirectory)
{
if (newDirectory?.Files == null)
return;

foreach (SlskdDownloadFile file in newDirectory.Files)
if (_previousFileStates.TryGetValue(file.Id, out string? previousState) && previousState != file.State)
FileStateChanged?.Invoke(this, file);
_previousFileStates = newDirectory.Files.ToDictionary(file => file.Id, file => file.State);
{
if (_previousFileStates.TryGetValue(file.Filename, out SlskdFileState? fileState) && fileState != null)
{
fileState.UpdateFile(file);
if (fileState.State != fileState.PreviousState)
FileStateChanged?.Invoke(this, fileState);
}
else
_previousFileStates.Add(file.Filename, new(file));


}
}

public OsPath GetFullFolderPath(string downloadPath) => new(Path.Combine(downloadPath, SlskdDownloadDirectory?.Directory
Expand Down Expand Up @@ -92,17 +105,17 @@ public DownloadClientItem GetDownloadClientItem(string downloadPath, TimeSpan? t
_lastUpdateTime = now;
_lastDownloadedSize = downloadedSize;

List<DownloadItemStatus> fileStatuses = SlskdDownloadDirectory.Files.Select(file => file.GetStatus()).ToList();
List<string> failedFiles = SlskdDownloadDirectory.Files
List<DownloadItemStatus> fileStatuses = _previousFileStates.Values.Select(file => file.GetStatus()).ToList();
List<string> failedFiles = _previousFileStates.Values
.Where(file => file.GetStatus() == DownloadItemStatus.Failed)
.Select(file => Path.GetFileName(file.Filename)).ToList();
.Select(file => Path.GetFileName(file.File.Filename)).ToList();

DownloadItemStatus status = DownloadItemStatus.Queued;
DateTime lastTime = SlskdDownloadDirectory.Files.Max(x => x.EnqueuedAt > x.StartedAt ? x.EnqueuedAt : x.StartedAt + x.ElapsedTime);

if (now - lastTime > timeout)
status = DownloadItemStatus.Failed;
else if ((double)failedFiles.Count / fileStatuses.Count * 100 > 10)
else if ((double)failedFiles.Count / fileStatuses.Count * 100 > 20)
{
status = DownloadItemStatus.Failed;
_downloadClientItem.Message = $"Downloading {failedFiles.Count} files failed: {string.Join(", ", failedFiles)}";
Expand All @@ -121,10 +134,13 @@ public DownloadClientItem GetDownloadClientItem(string downloadPath, TimeSpan? t
}
else if (fileStatuses.Any(status => status == DownloadItemStatus.Paused))
status = DownloadItemStatus.Paused;
else if (fileStatuses.Any(status => status == DownloadItemStatus.Downloading))
status = DownloadItemStatus.Downloading;
else if (fileStatuses.Any(status => status == DownloadItemStatus.Warning))
{
_downloadClientItem.Message = "Some files failed. Retrying download...";
status = DownloadItemStatus.Warning;
}
else if (fileStatuses.Any(status => status == DownloadItemStatus.Downloading))
status = DownloadItemStatus.Downloading;

// Update DownloadClientItem
_downloadClientItem.TotalSize = totalSize;
Expand All @@ -136,6 +152,58 @@ public DownloadClientItem GetDownloadClientItem(string downloadPath, TimeSpan? t
}
}

public class SlskdFileState
{
public SlskdDownloadFile File { get; private set; } = null!;
public int RetryCount { get; private set; }
private bool _retried = true;
public int MaxRetryCount { get; private set; } = 1;
public string State => File.State;
public string PreviousState { get; private set; } = "Requested";

public DownloadItemStatus GetStatus()
{
DownloadItemStatus status = GetStatus(State);
if ((status == DownloadItemStatus.Failed && RetryCount < MaxRetryCount) || _retried)
return DownloadItemStatus.Warning;
return status;
}

private static DownloadItemStatus GetStatus(string state) => state switch
{
"Requested" => DownloadItemStatus.Queued, // "Requested" is treated as "Queued"
"Queued, Remotely" or "Queued, Locally" => DownloadItemStatus.Queued, // Both are queued states
"Initializing" => DownloadItemStatus.Queued, // "Initializing" is treated as "Queued"
"InProgress" => DownloadItemStatus.Downloading, // "InProgress" maps to "Downloading"
"Completed, Succeeded" => DownloadItemStatus.Completed, // Successful completion
"Completed, Cancelled" => DownloadItemStatus.Failed, // Cancelled is treated as "Failed"
"Completed, TimedOut" => DownloadItemStatus.Failed, // Timed out is treated as "Failed"
"Completed, Errored" => DownloadItemStatus.Failed, // Errored is treated as "Failed"
"Completed, Rejected" => DownloadItemStatus.Failed, // Rejected is treated as "Failed"
_ => DownloadItemStatus.Queued // Default to "Queued" for unknown states
};

public SlskdFileState(SlskdDownloadFile file) => UpdateFile(file);

public void UpdateFile(SlskdDownloadFile file)
{
if (!_retried)
PreviousState = State;
else if (File != null && GetStatus(file.State) == DownloadItemStatus.Failed)
PreviousState = "Requested";
File = file;
_retried = false;
}

public void UpdateMaxRetryCount(int maxRetryCount) => MaxRetryCount = maxRetryCount;

public void IncrementAttempt()
{
_retried = true;
RetryCount++;
}
}

public record SlskdDownloadDirectory(string Directory, int FileCount, List<SlskdDownloadFile>? Files)
{
public static IEnumerable<SlskdDownloadDirectory> GetDirectories(JsonElement directoriesElement)
Expand Down Expand Up @@ -172,22 +240,8 @@ public record SlskdDownloadFile(
double PercentComplete,
TimeSpan RemainingTime,
TimeSpan? EndedAt
)
)
{
public DownloadItemStatus GetStatus() => State switch
{
"Requested" => DownloadItemStatus.Queued, // "Requested" is treated as "Queued"
"Queued, Remotely" or "Queued, Locally" => DownloadItemStatus.Queued, // Both are queued states
"Initializing" => DownloadItemStatus.Queued, // "Initializing" is treated as "Queued"
"InProgress" => DownloadItemStatus.Downloading, // "InProgress" maps to "Downloading"
"Completed, Succeeded" => DownloadItemStatus.Completed, // Successful completion
"Completed, Cancelled" => DownloadItemStatus.Failed, // Cancelled is treated as "Failed"
"Completed, TimedOut" => DownloadItemStatus.Failed, // Timed out is treated as "Failed"
"Completed, Errored" => DownloadItemStatus.Failed, // Errored is treated as "Failed"
"Completed, Rejected" => DownloadItemStatus.Failed, // Rejected is treated as "Failed"
_ => DownloadItemStatus.Queued // Default to "Queued" for unknown states
};

public static IEnumerable<SlskdDownloadFile> GetFiles(JsonElement filesElement)
{
if (filesElement.ValueKind != JsonValueKind.Array)
Expand Down
9 changes: 8 additions & 1 deletion Tubifarry/Download/Clients/Soulseek/SlskdProviderSettings.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,11 @@ public SlskdProviderSettingsValidator()
.GreaterThanOrEqualTo(0.1)
.WithMessage("Timeout must be at least 0.1 hours.")
.When(c => c.Timeout.HasValue);

// RetryAttempts validation
RuleFor(c => c.RetryAttempts)
.InclusiveBetween(0, 10)
.WithMessage("Retry attempts must be between 0 and 10.");
}
}

Expand All @@ -61,13 +66,15 @@ public class SlskdProviderSettings : IProviderConfig
[FieldDefinition(7, Label = "Timeout", Type = FieldType.Textbox, HelpText = "Specify the maximum time to wait for a response from the Slskd instance before timing out. Fractional values are allowed (e.g., 1.5 for 1 hour and 30 minutes). Set leave blank for no timeout.", Unit = "hours", Advanced = true, Placeholder = "Enter timeout in hours")]
public double? Timeout { get; set; }

[FieldDefinition(8, Label = "Retry Attempts", Type = FieldType.Number, HelpText = "The number of times to retry downloading a file if it fails.", Advanced = true, Placeholder = "Enter retry attempts")]
public int RetryAttempts { get; set; } = 1;

[FieldDefinition(98, Label = "Is Fetched remote", Type = FieldType.Checkbox, Hidden = HiddenType.Hidden)]
public bool IsRemotePath { get; set; }

[FieldDefinition(99, Label = "Is Localhost", Type = FieldType.Checkbox, Hidden = HiddenType.Hidden)]
public bool IsLocalhost { get; set; }


public TimeSpan? GetTimeout() => Timeout == null ? null : TimeSpan.FromHours(Timeout.Value);

public NzbDroneValidationResult Validate() => new(Validator.Validate(this));
Expand Down
2 changes: 2 additions & 0 deletions Tubifarry/Download/Clients/YouTube/YoutubeClient.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
using NzbDrone.Core.Organizer;
using NzbDrone.Core.Parser.Model;
using NzbDrone.Core.RemotePathMappings;
using Requests;
using Tubifarry.Core.Model;
using Tubifarry.Core.Utilities;
using Xabe.FFmpeg;
Expand All @@ -23,6 +24,7 @@ public YoutubeClient(IYoutubeDownloadManager dlManager, IConfigService configSer
{
_dlManager = dlManager;
_naminService = namingConfigService;
RequestHandler.MainRequestHandlers[1].MaxParallelism = 1;
}

public override string Name => "Youtube";
Expand Down
2 changes: 1 addition & 1 deletion Tubifarry/Download/Clients/YouTube/YoutubeProviderSettings.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@ public class YoutubeProviderSettings : IProviderConfig
[FieldDefinition(0, Label = "Download Path", Type = FieldType.Path, HelpText = "Specify the directory where downloaded files will be saved.")]
public string DownloadPath { get; set; } = "";

[FieldDefinition(1, Label = "Cookie Path", Type = FieldType.FilePath, Placeholder = "/downloads/Cookies/cookies.txt", HelpText = "Specify the path to the YouTube cookies file. This is optional but required for accessing restricted content.", Advanced = true)]
[FieldDefinition(1, Label = "Cookie Path", Type = FieldType.FilePath, Hidden = HiddenType.HiddenIfNotSet, Placeholder = "/downloads/Cookies/cookies.txt", HelpText = "Specify the path to the YouTube cookies file. This is optional but required for accessing restricted content.", Advanced = true)]
public string CookiePath { get; set; } = string.Empty;

[FieldDefinition(2, Label = "Use ID3v2.3 Tags", HelpText = "Enable this option to use ID3v2.3 tags for better compatibility with older media players like Windows Media Player.", Type = FieldType.Checkbox, Advanced = true)]
Expand Down
Loading

0 comments on commit 4977287

Please sign in to comment.