diff --git a/.gitignore b/.gitignore index 9491a2f..18ede69 100644 --- a/.gitignore +++ b/.gitignore @@ -360,4 +360,5 @@ MigrationBackup/ .ionide/ # Fody - auto-generated XML schema -FodyWeavers.xsd \ No newline at end of file +FodyWeavers.xsd +.idea/.idea.Mapperator/.idea/misc.xml diff --git a/Mapperator.ConsoleApp/DbManager.cs b/Mapperator.ConsoleApp/DbManager.cs index 7f2e5a2..4f57f44 100644 --- a/Mapperator.ConsoleApp/DbManager.cs +++ b/Mapperator.ConsoleApp/DbManager.cs @@ -10,6 +10,7 @@ using Mapping_Tools_Core.BeatmapHelper; using Mapping_Tools_Core.BeatmapHelper.HitObjects.Objects; using Mapping_Tools_Core.BeatmapHelper.IO.Editor; +using OsuParsers.Database; using OsuParsers.Enums; namespace Mapperator.ConsoleApp { @@ -30,10 +31,17 @@ public static IEnumerable GetCollection(string collectionName) { return collection.MD5Hashes.SelectMany(o => beatmaps.Where(b => b.MD5Hash == o)); } - public static List GetAll() { + public static OsuDatabase GetOsuDatabase() { var osuDbPath = Path.Join(ConfigManager.Config.OsuPath, "osu!.db"); - var db = DatabaseDecoder.DecodeOsu(osuDbPath); - return db.Beatmaps; + return DatabaseDecoder.DecodeOsu(osuDbPath); + } + + public static List GetAll() { + return GetOsuDatabase().Beatmaps; + } + + public static IEnumerable GetMapSet(OsuDatabase db, int setId) { + return db.Beatmaps.Where(o => o.BeatmapSetId == setId); } public static IEnumerable GetFiltered(IHasFilter opts) { @@ -46,9 +54,11 @@ public static bool DbBeatmapFilter(DbBeatmap o, IHasFilter opts) { var regex = new Regex(@$"(?!\s?(de\s)?(it|that|{string.Join('|', opts.Mapper!.Select(Regex.Escape))}))(((^|[^\S\r\n])(\S)*([sz]'|'s))|((^|[^\S\r\n])de\s(\S)*))", RegexOptions.IgnoreCase); return (!opts.MinId.HasValue || o.BeatmapSetId >= opts.MinId) + && (!opts.MaxId.HasValue || o.BeatmapSetId <= opts.MaxId) && (!opts.RankedStatus!.Any() || opts.RankedStatus!.Contains(o.RankedStatus)) && o.Ruleset == opts.Ruleset && (!opts.MinStarRating.HasValue || GetDefaultStarRating(o) >= opts.MinStarRating) + && (!opts.MaxStarRating.HasValue || GetDefaultStarRating(o) <= opts.MaxStarRating) && (!opts.Mapper!.Any() || (opts.Mapper!.Any(x => x == o.Creator || o.Difficulty.Contains(x)) && !o.Difficulty.Contains("Hitsounds", StringComparison.OrdinalIgnoreCase) && !o.Difficulty.Contains("Collab", StringComparison.OrdinalIgnoreCase) @@ -56,18 +66,20 @@ public static bool DbBeatmapFilter(DbBeatmap o, IHasFilter opts) { } public static double GetDefaultStarRating(DbBeatmap beatmap) { - return beatmap.Ruleset switch { - Ruleset.Taiko => beatmap.TaikoStarRating[Mods.None], - Ruleset.Mania => beatmap.ManiaStarRating[Mods.None], - Ruleset.Fruits => beatmap.CatchStarRating[Mods.None], - _ => beatmap.StandardStarRating[Mods.None] + var dict = beatmap.Ruleset switch { + Ruleset.Taiko => beatmap.TaikoStarRating, + Ruleset.Mania => beatmap.ManiaStarRating, + Ruleset.Fruits => beatmap.CatchStarRating, + _ => beatmap.StandardStarRating }; + + return dict.TryGetValue(Mods.None, out double value) ? value : double.NaN; } public static IEnumerable GetFilteredAndRead(IHasFilter opts) { return GetFiltered(opts) .Select(o => Path.Combine(ConfigManager.Config.SongsPath, o.FolderName.Trim(), o.FileName.Trim())) - .Where(o => { + .Where((o) => { if (File.Exists(o)) { Console.Write('.'); return true; @@ -87,6 +99,29 @@ public static IEnumerable GetFilteredAndRead(IHasFilter opts) { }).Where(ValidBeatmap)!; } + public static IEnumerable<(IBeatmap, DbBeatmap)> GetFilteredAndRead2(IHasFilter opts) { + return GetFiltered(opts) + .Select(o => (Path.Combine(ConfigManager.Config.SongsPath, o.FolderName.Trim(), o.FileName.Trim()), o)) + .Where(o => { + if (File.Exists(o.Item1)) { + Console.Write('.'); + return true; + } + + Console.WriteLine(Strings.CouldNotFindFile, o.Item1); + return false; + }) + .Select<(string, DbBeatmap), (IBeatmap?, DbBeatmap)>(o => { + try { + return (new BeatmapEditor(o.Item1).ReadFile(), o.Item2); + } + catch (Exception e) { + Console.WriteLine(Strings.ErrorReadingFile, o.Item1, e); + return (null, o.Item2); + } + }).Where(o => ValidBeatmap(o.Item1))!; + } + public static bool ValidBeatmap(IBeatmap? beatmap) { if (beatmap == null) return false; diff --git a/Mapperator.ConsoleApp/IHasFilter.cs b/Mapperator.ConsoleApp/IHasFilter.cs index f1f0397..4f692c5 100644 --- a/Mapperator.ConsoleApp/IHasFilter.cs +++ b/Mapperator.ConsoleApp/IHasFilter.cs @@ -7,8 +7,10 @@ namespace Mapperator.ConsoleApp; public interface IHasFilter { public string? CollectionName { get; } int? MinId { get; } + int? MaxId { get; } IEnumerable? RankedStatus { get; } Ruleset Ruleset { get; } double? MinStarRating { get; } + double? MaxStarRating { get; } IEnumerable? Mapper { get; } } \ No newline at end of file diff --git a/Mapperator.ConsoleApp/Program.cs b/Mapperator.ConsoleApp/Program.cs index c0b0e45..928e874 100644 --- a/Mapperator.ConsoleApp/Program.cs +++ b/Mapperator.ConsoleApp/Program.cs @@ -7,7 +7,7 @@ public static class Program { private static int Main(string[] args) { ConfigManager.LoadConfig(); - return Parser.Default.ParseArguments(args) + return Parser.Default.ParseArguments(args) .MapResult( (Count.CountOptions opts) => Count.DoDataCount(opts), (Extract.ExtractOptions opts) => Extract.DoDataExtraction(opts), @@ -15,6 +15,8 @@ private static int Main(string[] args) { (Convert.ConvertOptions opts) => Convert.DoMapConvert(opts), (Search.SearchOptions opts) => Search.DoPatternSearch(opts), (Analyze.AnalyzeOptions opts) => Analyze.DoVisualSpacingExtract(opts), + (Extract2.Extract2Options opts) => Extract2.DoDataExtraction(opts), + (Dataset.DatasetOptions opts) => Dataset.DoDataExtraction(opts), (ConvertML.ConvertMLOptions opts) => ConvertML.DoMapConvert(opts), _ => 1); } diff --git a/Mapperator.ConsoleApp/Properties/launchSettings.json b/Mapperator.ConsoleApp/Properties/launchSettings.json index d434d7d..91caa1f 100644 --- a/Mapperator.ConsoleApp/Properties/launchSettings.json +++ b/Mapperator.ConsoleApp/Properties/launchSettings.json @@ -1,12 +1,16 @@ { "profiles": { + "CreateDataSet": { + "commandName": "Project", + "commandLineArgs": "dataset -m Standard -s Ranked -i 20000 -x 25000 -r 4 -o \"D:\\Osu! Dingen\\Beatmap ML Datasets\\old_test\"" + }, "ConvertMapML": { "commandName": "Project", "commandLineArgs": "convert-ml -m \"Resources\\sdf_osu_5_fine_2.h5\" -i \"Resources\\input\" -o output" }, "CountData": { "commandName": "Project", - "commandLineArgs": "count -s Ranked -m Standard -a Sotarks -v" + "commandLineArgs": "count -m Standard -s Ranked -i 20000 -x 25000 -r 4 -f" }, "VisualSpacingTest": { "commandName": "Project", @@ -20,6 +24,10 @@ "commandName": "Project", "commandLineArgs": "extract -s Ranked -m Standard -a Sotarks -o SotarksData" }, + "ExtractAllData122123": { + "commandName": "Project", + "commandLineArgs": "extract2 -s Ranked -m Standard -o pp_data_v2" + }, "ConvertMap": { "commandName": "Project", "commandLineArgs": "convert -d test -i \"Resources\\input\" -o output" diff --git a/Mapperator.ConsoleApp/Resources/Strings.Designer.cs b/Mapperator.ConsoleApp/Resources/Strings.Designer.cs index b22c9bd..221fee3 100644 --- a/Mapperator.ConsoleApp/Resources/Strings.Designer.cs +++ b/Mapperator.ConsoleApp/Resources/Strings.Designer.cs @@ -98,6 +98,60 @@ internal static string CouldNotFindFile { } } + /// + /// Looks up a localized string similar to Total duration: {0}. + /// + internal static string Count_DoDataCount_Total_duration___0_ { + get { + return ResourceManager.GetString("Count_DoDataCount_Total_duration___0_", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to Total file size: {0} MB. + /// + internal static string Count_DoDataCount_Total_file_size___0__MB { + get { + return ResourceManager.GetString("Count_DoDataCount_Total_file_size___0__MB", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to Copied beatmap set {0}/{1}. + /// + internal static string Dataset_DoDataExtraction_Copy_Update { + get { + return ResourceManager.GetString("Dataset_DoDataExtraction_Copy_Update", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to {0} sets found. + /// + internal static string Dataset_DoDataExtraction_Count_Update { + get { + return ResourceManager.GetString("Dataset_DoDataExtraction_Count_Update", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to Finding beatmap sets.... + /// + internal static string Dataset_DoDataExtraction_Finding_beatmap_sets___ { + get { + return ResourceManager.GetString("Dataset_DoDataExtraction_Finding_beatmap_sets___", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to Writing dataset.... + /// + internal static string Dataset_DoDataExtraction_Writing_dataset___ { + get { + return ResourceManager.GetString("Dataset_DoDataExtraction_Writing_dataset___", resourceCulture); + } + } + /// /// Looks up a localized string similar to Error reading file '{0}': ///{1}. diff --git a/Mapperator.ConsoleApp/Resources/Strings.resx b/Mapperator.ConsoleApp/Resources/Strings.resx index 393e0ce..91df9b2 100644 --- a/Mapperator.ConsoleApp/Resources/Strings.resx +++ b/Mapperator.ConsoleApp/Resources/Strings.resx @@ -172,6 +172,23 @@ Basic usage: Converting spacing to reference beatmap... + + Total file size: {0} MB + + + Total duration: {0} + + + Finding beatmap sets... + + + {0} sets found + + + Writing dataset... + + + Copied beatmap set {0}/{1} Loading ML model... diff --git a/Mapperator.ConsoleApp/Verbs/Count.cs b/Mapperator.ConsoleApp/Verbs/Count.cs index 98bef97..2ba653c 100644 --- a/Mapperator.ConsoleApp/Verbs/Count.cs +++ b/Mapperator.ConsoleApp/Verbs/Count.cs @@ -1,24 +1,61 @@ using System; +using System.Collections.Generic; +using System.IO; using System.Linq; using CommandLine; using JetBrains.Annotations; using Mapperator.ConsoleApp.Resources; +using Mapping_Tools_Core.Audio; namespace Mapperator.ConsoleApp.Verbs; public static class Count { [Verb("count", HelpText = "Count the amount of beatmaps available matching the specified filter.")] public class CountOptions : FilterBase { + [Option('u', "uniqueSong", HelpText = "Count each unique song file", Default = false)] + public bool UniqueSong { get; [UsedImplicitly] set; } + + [Option('f', "fileSize", HelpText = "Aggregate the filesize of the songs", Default = false)] + public bool FileSize { get; [UsedImplicitly] set; } + [Option('v', "verbose", HelpText = "Print the name of each counted beatmap", Default = false)] public bool Verbose { get; [UsedImplicitly] set; } } public static int DoDataCount(CountOptions opts) { + var songNames = new HashSet(); + long totalSize = 0; + var totalTime = TimeSpan.Zero; + Console.WriteLine(DbManager.GetFiltered(opts) - .Count(o => { if (opts.Verbose) Console.WriteLine(Strings.FullBeatmapName, o.Artist, o.Title, o.Creator, o.Difficulty); + .Count(o => { + if (opts.Verbose) Console.WriteLine(Strings.FullBeatmapName, o.Artist, o.Title, o.Creator, o.Difficulty); + if (!opts.UniqueSong) return true; + string songFile = Path.Combine(ConfigManager.Config.SongsPath, o.FolderName.Trim(), o.AudioFileName.Trim()); + string songName = $"{o.Artist} - {Dataset.RemovePartsBetweenParentheses(o.Title)}"; + if (!string.Equals(Path.GetExtension(songFile), ".mp3", StringComparison.OrdinalIgnoreCase)) return false; + if (songNames.Contains(songName)) return false; + songNames.Add(songName); + var info = new FileInfo(songFile); + if (!info.Exists) return false; + if (opts.FileSize) { + totalSize += info.Length; + try { + totalTime += new Mp3FileReader(songFile).TotalTime; + } catch (InvalidOperationException e) { + Console.WriteLine(e); + return false; + } + } + if (opts.Verbose) Console.WriteLine(songName); return true; })); + if (opts.FileSize) { + Console.WriteLine(Strings.Count_DoDataCount_Total_file_size___0__MB, totalSize / 1024 / 1024); + Console.WriteLine(Strings.Count_DoDataCount_Total_duration___0_, totalTime); + } + return 0; } } \ No newline at end of file diff --git a/Mapperator.ConsoleApp/Verbs/Dataset.cs b/Mapperator.ConsoleApp/Verbs/Dataset.cs new file mode 100644 index 0000000..5004dc4 --- /dev/null +++ b/Mapperator.ConsoleApp/Verbs/Dataset.cs @@ -0,0 +1,274 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Text.Json; +using System.Text.Json.Serialization; +using CommandLine; +using JetBrains.Annotations; +using Mapperator.ConsoleApp.Resources; +using Mapping_Tools_Core.Audio; +using Mapping_Tools_Core.BeatmapHelper; +using Mapping_Tools_Core.BeatmapHelper.IO.Editor; +using NVorbis; +using OsuParsers.Database.Objects; +using OsuParsers.Enums; +using OsuParsers.Enums.Database; + +namespace Mapperator.ConsoleApp.Verbs; + +public static class Dataset { + [Verb("dataset", HelpText = "Extract a ML dataset from your osu! database.")] + public class DatasetOptions : FilterBase { + [Option('o', "output", Required = true, HelpText = "Folder to output the dataset to.")] + public string? OutputFolder { get; [UsedImplicitly] set; } + } + + public static int DoDataExtraction(DatasetOptions opts) { + if (opts.OutputFolder is null) throw new ArgumentNullException(nameof(opts)); + + if (!Directory.Exists(opts.OutputFolder)) { + Directory.CreateDirectory(opts.OutputFolder); + } + + Console.WriteLine(Strings.Dataset_DoDataExtraction_Finding_beatmap_sets___); + + var mapSets = new List<(int, string)>(); + var mapSetIds = new HashSet(); + var mapMd5Hashes = new HashSet(); + long totalSize = 0; + var totalTime = TimeSpan.Zero; + + foreach (var o in DbManager.GetFiltered(opts)) { + string songFile = Path.Combine(ConfigManager.Config.SongsPath, o.FolderName.Trim(), o.AudioFileName.Trim()); + string mapFile = Path.Combine(ConfigManager.Config.SongsPath, o.FolderName.Trim(), o.FileName.Trim()); + string extension = Path.GetExtension(songFile).ToLower(); + + mapMd5Hashes.Add(o.MD5Hash); + + if (mapSetIds.Contains(o.BeatmapSetId)) continue; + + var info = new FileInfo(songFile); + if (!info.Exists) continue; + + totalSize += info.Length; + try { + switch (extension) { + case ".mp3": + totalTime += new Mp3FileReader(songFile).TotalTime; + break; + case ".ogg": + totalTime += new VorbisReader(songFile).TotalTime; + break; + default: + continue; + } + } catch (InvalidOperationException e) { + Console.WriteLine(e); + continue; + } + + if (!File.Exists(mapFile)) continue; + try { + new BeatmapEditor(mapFile).ReadFile(); + } catch (Exception e) { + Console.WriteLine(Strings.ErrorReadingFile, mapFile, e); + continue; + } + + mapSets.Add((o.BeatmapSetId, songFile)); + mapSetIds.Add(o.BeatmapSetId); + Console.Write('\r'); + Console.Write(Strings.Dataset_DoDataExtraction_Count_Update, mapSets.Count); + } + + Console.WriteLine(); + Console.WriteLine(Strings.Count_DoDataCount_Total_file_size___0__MB, totalSize / 1024 / 1024); + Console.WriteLine(Strings.Count_DoDataCount_Total_duration___0_, totalTime); + Console.WriteLine(Strings.Dataset_DoDataExtraction_Writing_dataset___); + + const string mapSubFolder = "beatmaps"; + const string audioName = "audio"; + const string metadataName = "metadata.json"; + var options = new JsonSerializerOptions { WriteIndented = true, Converters = { new DictionaryConverter() }}; + + var sortedMapSets = mapSets.OrderBy(o => o.Item1).ToArray(); + var db = DbManager.GetOsuDatabase(); + int mapSetCount = 0; + int totalBeatmapCount = 0; + for (var i = 0; i < sortedMapSets.Length; i++) { + var (mapSetId, songFile) = sortedMapSets[i]; + var maps = DbManager.GetMapSet(db, mapSetId).OrderBy(DbManager.GetDefaultStarRating).ToArray(); + if (maps.Length == 0) continue; + + string mapSetFolderName = $"Track{mapSetCount:D5}"; + string mapSetFolderPath = Path.Combine(opts.OutputFolder, mapSetFolderName); + Directory.CreateDirectory(mapSetFolderPath); + Directory.CreateDirectory(Path.Combine(mapSetFolderPath, mapSubFolder)); + + var mapCount = 0; + DbBeatmap? lastMap = null; + var beatmapMetadatas = new Dictionary(); + foreach (var dbBeatmap in maps) { + if (!DbManager.DbBeatmapFilter(dbBeatmap, opts) || !mapMd5Hashes.Contains(dbBeatmap.MD5Hash)) continue; + + string mapFile = Path.Combine(ConfigManager.Config.SongsPath, dbBeatmap.FolderName.Trim(), dbBeatmap.FileName.Trim()); + if (!File.Exists(mapFile)) continue; + try { + var editor = new BeatmapEditor(mapFile); + var beatmap = editor.ReadFile(); + ClearStoryboard(beatmap.Storyboard); + + // Make sure the beatmap ID matches the one in the database + beatmap.Metadata.BeatmapId = dbBeatmap.BeatmapId; + beatmap.Metadata.BeatmapSetId = dbBeatmap.BeatmapSetId; + + string mapName = $"{totalBeatmapCount:D6}M{mapCount:D3}"; + string mapOutputName = Path.Combine(opts.OutputFolder, mapSetFolderName, mapSubFolder, mapName + ".osu"); + editor.Path = mapOutputName; + editor.WriteFile(beatmap); + + beatmapMetadatas.Add(mapName, new BeatmapMetadata( + totalBeatmapCount, + dbBeatmap.BeatmapId, + dbBeatmap.Ruleset, + dbBeatmap.MD5Hash, + dbBeatmap.Difficulty, + dbBeatmap.OnlineOffset, + dbBeatmap.DrainTime, + dbBeatmap.TotalTime, + dbBeatmap.RankedStatus, + dbBeatmap.CirclesCount, + dbBeatmap.SpinnersCount, + dbBeatmap.SlidersCount, + dbBeatmap.CircleSize, + dbBeatmap.ApproachRate, + dbBeatmap.OverallDifficulty, + dbBeatmap.HPDrain, + dbBeatmap.SliderVelocity, + dbBeatmap.StackLeniency, + dbBeatmap.StandardStarRating, + dbBeatmap.TaikoStarRating, + dbBeatmap.CatchStarRating, + dbBeatmap.ManiaStarRating + )); + + mapCount++; + totalBeatmapCount++; + lastMap = dbBeatmap; + } catch (Exception e) { + Console.WriteLine(Strings.ErrorReadingFile, mapFile, e); + } + } + + if (lastMap is null) { + // There are no valid maps for this song so delete the mapset folder + Directory.Delete(mapSetFolderPath, true); + continue; + } + + // Write metadata + string audioNameWithExtension = audioName + Path.GetExtension(songFile).ToLower(); + var metadata = new Metadata( + mapSetId, + lastMap.Artist, + lastMap.Title, + lastMap.Source, + lastMap.Creator, + lastMap.Tags, + audioNameWithExtension, + mapSubFolder, + beatmapMetadatas + ); + string json = JsonSerializer.Serialize(metadata, options); + File.WriteAllText(Path.Combine(mapSetFolderPath, metadataName), json); + + // Copy audio file + File.Copy(songFile, Path.Combine(opts.OutputFolder, mapSetFolderName, audioNameWithExtension), true); + + Console.Write('\r'); + Console.Write(Strings.Dataset_DoDataExtraction_Copy_Update, i + 1, sortedMapSets.Length); + mapSetCount++; + } + + return 0; + } + + private static void ClearStoryboard(IStoryboard sb) { + sb.BackgroundColourTransformations.Clear(); + sb.StoryboardLayerBackground.Clear(); + sb.StoryboardLayerFail.Clear(); + sb.StoryboardLayerForeground.Clear(); + sb.StoryboardLayerOverlay.Clear(); + sb.StoryboardLayerPass.Clear(); + sb.StoryboardSoundSamples.Clear(); + sb.BackgroundAndVideoEvents.Clear(); + } + + public static string RemovePartsBetweenParentheses(string str) { + Span span = stackalloc char[str.Length]; + int j = 0; + for (int i = 0; i < str.Length; i++) { + if (str[i] == '(') { + if (i > 0 && str[i - 1] == ' ') j--; + while (str[i] != ')') i++; + continue; + } + span[j++] = str[i]; + } + return span[..j].ToString(); + } + + private class DictionaryConverter : JsonConverter> { + public override Dictionary Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) { + throw new NotImplementedException(); + } + + public override void Write(Utf8JsonWriter writer, Dictionary dictionary, JsonSerializerOptions options) { + writer.WriteStartObject(); + foreach ((var key, double value) in dictionary) { + writer.WritePropertyName(((int)key).ToString(CultureInfo.InvariantCulture)); + writer.WriteNumberValue(value); + } + writer.WriteEndObject(); + } + } + + [SuppressMessage("ReSharper", "NotAccessedPositionalProperty.Local")] + private record Metadata( + int BeatmapSetId, + string Artist, + string Title, + string Source, + string Creator, + string Tags, + string AudioFile, + string MapDir, + Dictionary Beatmaps); + + [SuppressMessage("ReSharper", "NotAccessedPositionalProperty.Local")] + private record BeatmapMetadata(int Index, + int BeatmapId, + Ruleset Ruleset, + string MD5Hash, + string Difficulty, + short OnlineOffset, + int DrainTime, + int TotalTime, + RankedStatus RankedStatus, + int CirclesCount, + int SpinnersCount, + int SlidersCount, + float CircleSize, + float ApproachRate, + float OverallDifficulty, + float HPDrain, + double SliderVelocity, + float StackLeniency, + Dictionary StandardStarRating, + Dictionary TaikoStarRating, + Dictionary CatchStarRating, + Dictionary ManiaStarRating); +} \ No newline at end of file diff --git a/Mapperator.ConsoleApp/Verbs/Extract2.cs b/Mapperator.ConsoleApp/Verbs/Extract2.cs new file mode 100644 index 0000000..d912e6c --- /dev/null +++ b/Mapperator.ConsoleApp/Verbs/Extract2.cs @@ -0,0 +1,28 @@ +using System; +using System.IO; +using System.Linq; +using CommandLine; +using JetBrains.Annotations; + +namespace Mapperator.ConsoleApp.Verbs; + +public static class Extract2 { + [Verb("extract2", HelpText = "Extract beatmap data from an osu! collection to ML data.")] + public class Extract2Options : FilterBase { + [Option('o', "output", Required = true, HelpText = "Filename of the output.")] + public string? OutputName { get; [UsedImplicitly] set; } + } + + public static int DoDataExtraction(Extract2Options opts) { + if (opts.OutputName is null) throw new ArgumentNullException(nameof(opts)); + + bool[] mirrors = { false, true }; + var extractor = new DataExtractor2(); + File.WriteAllLines(Path.ChangeExtension(opts.OutputName, ".txt"), + DataSerializer2.SerializeBeatmapData(DbManager.GetFilteredAndRead2(opts) + .Select(b => (extractor.ExtractBeatmapData(b.Item1), b.Item1.Difficulty, b.Item2.BeatmapId)) + )); + + return 0; + } +} \ No newline at end of file diff --git a/Mapperator.ConsoleApp/Verbs/FilterBase.cs b/Mapperator.ConsoleApp/Verbs/FilterBase.cs index 8f4bb77..8987a76 100644 --- a/Mapperator.ConsoleApp/Verbs/FilterBase.cs +++ b/Mapperator.ConsoleApp/Verbs/FilterBase.cs @@ -13,15 +13,21 @@ public abstract class FilterBase : IHasFilter { [Option('i', "minId", HelpText = "Filter the minimum beatmap set ID.")] public int? MinId { get; [UsedImplicitly] set; } + [Option('x', "maxId", HelpText = "Filter the maximum beatmap set ID.")] + public int? MaxId { get; [UsedImplicitly] set; } + [Option('s', "status", HelpText = "Filter the ranked status.", Separator = ',')] public IEnumerable? RankedStatus { get; [UsedImplicitly] set; } [Option('m', "mode", HelpText = "Filter the game mode.", Default = Ruleset.Standard)] public Ruleset Ruleset { get; [UsedImplicitly] set; } - [Option('r', "starRating", HelpText = "Filter the star rating.")] + [Option('r', "minStarRating", HelpText = "Filter the minimum star rating.")] public double? MinStarRating { get; [UsedImplicitly] set; } + [Option('t', "maxStarRating", HelpText = "Filter the maximum star rating.")] + public double? MaxStarRating { get; [UsedImplicitly] set; } + [Option('a', "mapper", HelpText = "Filter on mapper name.", Separator = ',')] public IEnumerable? Mapper { get; [UsedImplicitly] set; } } \ No newline at end of file diff --git a/Mapperator/DataExtractor2.cs b/Mapperator/DataExtractor2.cs new file mode 100644 index 0000000..57e115b --- /dev/null +++ b/Mapperator/DataExtractor2.cs @@ -0,0 +1,94 @@ +using Mapperator.Model; +using Mapping_Tools_Core.BeatmapHelper; +using Mapping_Tools_Core.BeatmapHelper.Contexts; +using Mapping_Tools_Core.BeatmapHelper.Enums; +using Mapping_Tools_Core.BeatmapHelper.HitObjects; +using Mapping_Tools_Core.BeatmapHelper.HitObjects.Objects; +using Mapping_Tools_Core.BeatmapHelper.IO.Encoding.HitObjects; +using Mapping_Tools_Core.BeatmapHelper.TimingStuff; +using Mapping_Tools_Core.MathUtil; + +namespace Mapperator { + public class DataExtractor2 { + public IEnumerable ExtractBeatmapData(IBeatmap beatmap, bool mirror = false) { + return ExtractBeatmapData(beatmap.HitObjects, beatmap.BeatmapTiming, beatmap.Difficulty.SliderTickRate, mirror); + } + + public IEnumerable ExtractBeatmapData(IEnumerable hitobjects, Timing timing, double sliderTickRate, bool mirror = false) { + var lastPos = new Vector2(256, 192); // Playfield centre + var lastLastPos = new Vector2(0, 192); // Playfield left-centre + int lastTime = 0; + foreach (var ho in hitobjects) { + switch (ho) { + case HitCircle: + yield return CreateDataPoint(ho.Pos, (int)ho.StartTime, DataType2.HitCircle, ref lastLastPos, ref lastPos, ref lastTime, mirror); + break; + case Slider slider: + yield return CreateDataPoint(ho.Pos, (int)ho.StartTime, DataType2.SliderStart, ref lastLastPos, ref lastPos, ref lastTime, mirror); + + // Get middle and end positions too for every repeat + var path = slider.GetSliderPath(); + var startPos = slider.Pos; + //var middlePos = path.PositionAt(0.5); + + if (!ho.TryGetContext(out var timing2)) { + throw new InvalidOperationException("Slider is not initialized with timing context. Can not get the slider ticks."); + } + + var t = timing2.UninheritedTimingPoint.MpB / sliderTickRate; + var tick_ts = new List(); + while (t + 10 < slider.SpanDuration) { + var t2 = t / slider.SpanDuration; + tick_ts.Add(t2); + + t += timing2.UninheritedTimingPoint.MpB / sliderTickRate; + } + + var endPos = path.PositionAt(1); + + for (int i = 0; i < slider.SpanCount; i++) { + // Do ticks + for (int j = 0; j < tick_ts.Count; j++) { + var k = i % 2 == 0 ? j : tick_ts.Count - j - 1; + var t2 = tick_ts[k]; + var pos = path.PositionAt(t2); + var time = (int)(slider.StartTime + i * slider.SpanDuration + (i % 2 == 0 ? t2 : 1 - t2) * slider.SpanDuration); + + yield return CreateDataPoint(pos, time, DataType2.SliderTick, ref lastLastPos, ref lastPos, ref lastTime, mirror); + } + + // Do end + yield return CreateDataPoint(i % 2 == 0 ? endPos : startPos, (int)(slider.StartTime + slider.SpanDuration * (i + 1)), i == slider.RepeatCount ? DataType2.SliderEnd : DataType2.SliderRepeat, ref lastLastPos, ref lastPos, ref lastTime, mirror); + } + + break; + case Spinner spinner: + yield return CreateDataPoint(ho.Pos, (int)ho.StartTime, DataType2.SpinStart, ref lastLastPos, ref lastPos, ref lastTime, mirror); + yield return CreateDataPoint(ho.Pos, (int)spinner.EndTime, DataType2.SpinEnd, ref lastLastPos, ref lastPos, ref lastTime, mirror); + break; + } + } + } + + private MapDataPoint2 CreateDataPoint(Vector2 pos, int time, DataType2 dataType, ref Vector2 lastLastPos, ref Vector2 lastPos, ref int lastTime, bool mirror = false) { + //var angle = Vector2.Angle(pos - lastPos, lastPos - lastLastPos); + var angle = Helpers.AngleDifference((lastPos - lastLastPos).Theta, (pos - lastPos).Theta); + if (double.IsNaN(angle)) { + angle = 0; + } + + var point = new MapDataPoint2( + dataType, + time - lastTime, + Vector2.Distance(pos, lastPos), + mirror ? -angle : angle + ); + + lastLastPos = lastPos; + lastPos = pos; + lastTime = time; + + return point; + } + } +} diff --git a/Mapperator/DataSerializer2.cs b/Mapperator/DataSerializer2.cs new file mode 100644 index 0000000..fa199ae --- /dev/null +++ b/Mapperator/DataSerializer2.cs @@ -0,0 +1,44 @@ +using System.Globalization; +using Mapperator.Model; +using Mapping_Tools_Core.BeatmapHelper.Enums; +using Mapping_Tools_Core.BeatmapHelper.Sections; +using MoreLinq; + +namespace Mapperator { + public static class DataSerializer2 { + private const string BeatmapSeparator = "/-\\_/-\\_/-\\"; + + public static IEnumerable SerializeBeatmapData(IEnumerable<(IEnumerable, SectionDifficulty, int)> data) { + foreach (var (beatmap, difficulty, beatmapId) in data) { + yield return beatmapId.ToString(CultureInfo.InvariantCulture); + yield return difficulty.ApproachRate.ToString(CultureInfo.InvariantCulture); + yield return difficulty.CircleSize.ToString(CultureInfo.InvariantCulture); + yield return difficulty.OverallDifficulty.ToString(CultureInfo.InvariantCulture); + + foreach (var dataPoint in beatmap) { + yield return SerializeBeatmapDataSample(dataPoint); + } + + yield return BeatmapSeparator; + } + } + + public static string SerializeBeatmapDataSample(MapDataPoint2 data) { + return data.ToString(); + } + + public static IEnumerable> DeserializeBeatmapData(IEnumerable data) { + return data.Split(BeatmapSeparator, beatmapData => beatmapData.Select(DeserializeBeatmapDataSample)); + } + + public static MapDataPoint2 DeserializeBeatmapDataSample(string data) { + var split = data.Split(' '); + return new MapDataPoint2( + (DataType2)int.Parse(split[0], CultureInfo.InvariantCulture), + int.Parse(split[1], CultureInfo.InvariantCulture), + double.Parse(split[2], CultureInfo.InvariantCulture), + double.Parse(split[3], CultureInfo.InvariantCulture) + ); + } + } +} diff --git a/Mapperator/Model/DataType2.cs b/Mapperator/Model/DataType2.cs new file mode 100644 index 0000000..0ea7244 --- /dev/null +++ b/Mapperator/Model/DataType2.cs @@ -0,0 +1,11 @@ +namespace Mapperator.Model { + public enum DataType2 { + HitCircle, + SpinStart, + SpinEnd, + SliderStart, + SliderTick, + SliderRepeat, + SliderEnd + } +} \ No newline at end of file diff --git a/Mapperator/Model/MapDataPoint2.cs b/Mapperator/Model/MapDataPoint2.cs new file mode 100644 index 0000000..4c5fc17 --- /dev/null +++ b/Mapperator/Model/MapDataPoint2.cs @@ -0,0 +1,21 @@ +using System.Globalization; + +namespace Mapperator.Model { + public struct MapDataPoint2 { + public DataType2 DataType; + public int TimeSince; // The number of beats since the last data point + public double Spacing; // The distance from the previous to this point + public double Angle; // The angle between the vectors to the previous and previous previous points + + public MapDataPoint2(DataType2 dataType, int timeSince, double spacing, double angle) { + DataType = dataType; + TimeSince = timeSince; + Spacing = spacing; + Angle = angle; + } + + public override string ToString() { + return $"{((int)DataType).ToString(CultureInfo.InvariantCulture)} {TimeSince.ToString(CultureInfo.InvariantCulture)} {Spacing.ToString("N0", CultureInfo.InvariantCulture)} {Angle.ToString("N4", CultureInfo.InvariantCulture)}"; + } + } +}