2024.10.24.0

YT
YouTubeSettings: add 'DefaultVideoAllowWebm' and 'DefaultAudioEmbedThumbnail_Cover' settings
YouTubeMediaContainerBase: change cover selection for music download; fix adding incorrect playlist lines; allow 'webm' formats is there are no 'mp4' formats via http protocol

SCrawler
DeclaredNames: add new names
UserDataBase: add '_ForceSaveUserInfoOnException' field  and 'UpdateUserInformation_Ex' function to update user info on exception; clear '_MD5List' when clearing data and/or history
API.Instagram: add manual 'UserName' changing; mark user as non-existent if user ID cannot be obtained
API.Twitter: add manual 'UserName' changing
API.Mastodon: bypass inherited property
API.Reddit: fix incorrect UNIX date parsing
DownloadFeedForm: add exception handling to the 'RefillAfterDelete' function
MainFrame: add 'MENU_INFO_USER_SEARCH' to the 'Info' menu
SettingsHostCollection: fix a bug when changing data paths

Author: AAndyProgram

Changelog.md

@@ -1,3 +1,28 @@
+# 2024.10.24.0
+
+*2024-10-24*
+
+- Added
+  - YouTube (standalone app)
+    - settings `Embed thumbnail (cover)` and `Allow webm formats`
+    - changed cover selection for music downloads
+    - allow `webm` formats is there are no `mp4` formats via http protocol (issue #211)
+  - Sites:
+    - Instagram
+      - **ability to manually change username**
+      - **mark user as non-existent if user `ID` cannot be obtained**
+    - Twitter: **ability to manually change username**
+  - Main window: add users search button to 'Info' menu
+  - Minor improvements
+- Updated
+  - yt-dlp up to version **2024.10.22**
+  - gallery-dl up to version **1.27.6**
+- Fixed
+  - YouTube (standalone app): adding incorrect playlist lines
+  - Reddit: incorrect UNIX date parsing
+  - Can't change data path (issue #206)
+  - Minor bugs
+
 # 2024.9.2.0
 
 *2024-09-02*

FAQ.md

@@ -26,7 +26,7 @@ I strongly recommend you to **regularly** create backup copies of the settings f
 
 # Most frequently questions about SCrawler
 
-**Is something doesn't download, always check the [SITE'S REQUIREMENTS](https://github.com/AAndyProgram/SCrawler/wiki/Settings#sites-requirements) before asking questions!**
+**If something doesn't download, always check the [SITE'S REQUIREMENTS](https://github.com/AAndyProgram/SCrawler/wiki/Settings#sites-requirements) before asking questions!**
 
 *How to use: find your problem in the list and read the answer.*
 

SCrawler.YouTube/Base/YouTubeSettings.vb

@@ -311,6 +311,9 @@ Namespace API.YouTube.Base
         <Browsable(True), GridVisible, XMLVN({"DefaultsVideo"}, 1080), Category("Defaults Video"), DisplayName("Default definition"),
             Description("The default maximum video resolution. -1 for max definition")>
         Public ReadOnly Property DefaultVideoDefinition As XMLValue(Of Integer)
+        <Browsable(True), GridVisible, XMLVN({"DefaultsVideo"}, True), Category("Defaults Video"), DisplayName("Allow webm formats"),
+            Description("Allow webm formats over http if mp4 formats are not available. Default: true.")>
+        Public ReadOnly Property DefaultVideoAllowWebm As XMLValue(Of Boolean)
         <Browsable(True), GridVisible, XMLVN({"DefaultsVideo"}), Category("Defaults Video"), DisplayName("Convert non-AVC codecs to AVC"),
             Description("Convert non-AVC codecs (eg 'VP9') to AVC. Not recommended due to high CPU usage!")>
         Public ReadOnly Property DefaultVideoConvertNonAVC As XMLValue(Of Boolean)
@@ -416,6 +419,9 @@ Namespace API.YouTube.Base
         <Browsable(True), GridVisible, XMLVN({"DefaultsAudio"}, True), Category("Defaults Audio"), DisplayName("Embed thumbnail"),
             Description("Embed thumbnail in the audio as cover art. Default: true.")>
         Public ReadOnly Property DefaultAudioEmbedThumbnail As XMLValue(Of Boolean)
+        <Browsable(True), GridVisible, XMLVN({"DefaultsAudio"}, True), Category("Defaults Audio"), DisplayName("Embed thumbnail (cover)"),
+            Description("Try embedding the playlist cover (if it exists) as cover art. Default: true.")>
+        Public ReadOnly Property DefaultAudioEmbedThumbnail_Cover As XMLValue(Of Boolean)
         <Browsable(True), GridVisible, XMLVN({"DefaultsAudio"}, True), Category("Defaults Audio"), DisplayName("Embed thumbnail (extracted files)"),
             Description("Embed thumbnail in the extracted (additional file ('mp3' only)) audio as cover art. Default: true.")>
         Public ReadOnly Property DefaultAudioEmbedThumbnail_ExtractedFiles As XMLValue(Of Boolean)

SCrawler.YouTube/Declarations.vb

@@ -59,6 +59,7 @@ Namespace API.YouTube
         Public ReadOnly TrueUrlRegEx As RParams = RParams.DM(Base.YouTubeFunctions.TrueUrlPattern, 0, EDP.ReturnValue)
         Friend ReadOnly MusicUrlApply As RParams = RParams.DMS("https://([w\.]*)youtube.com.+", 1, RegexReturn.Replace, EDP.ReturnValue,
                                                                CType(Function(input$) "music.", Func(Of String, String)), String.Empty)
+        Friend ReadOnly M3U8ExcludedSymbols As String() = {".", ",", ":", "/", "\", "(", ")", "[", "]"}
         <Extension> Friend Function ToMusicUrl(ByVal URL As String, ByVal IsMusic As Boolean) As String
             Try : Return If(IsMusic And Not URL.IsEmptyString, CStr(RegexReplace(URL, MusicUrlApply)).IfNullOrEmpty(URL), URL) : Catch : Return URL : End Try
         End Function

SCrawler.YouTube/My Project/AssemblyInfo.vb

@@ -32,6 +32,6 @@ Imports System.Runtime.InteropServices
 ' by using the '*' as shown below:
 ' <Assembly: AssemblyVersion("1.0.*")>
 
-<Assembly: AssemblyVersion("2024.8.10.0")>
-<Assembly: AssemblyFileVersion("2024.8.10.0")>
+<Assembly: AssemblyVersion("2024.10.24.0")>
+<Assembly: AssemblyFileVersion("2024.10.24.0")>
 <Assembly: NeutralResourcesLanguage("en")>

SCrawler.YouTube/Objects/YouTubeMediaContainerBase.vb

@@ -175,7 +175,9 @@ Namespace API.YouTube.Objects
         Protected _ThumbnailUrl As String = String.Empty
         <XMLEC> Public Overridable Property ThumbnailUrl As String Implements IDownloadableMedia.ThumbnailUrl
             Get
-                If _ThumbnailUrl.IsEmptyString And Thumbnails.Count > 0 Then
+                If Not CoverURL.IsEmptyString Then
+                    Return CoverURL
+                ElseIf _ThumbnailUrl.IsEmptyString And Thumbnails.Count > 0 Then
                     Return Thumbnails.FirstOrDefault.URL
                 Else
                     Return _ThumbnailUrl
@@ -904,7 +906,8 @@ Namespace API.YouTube.Objects
             Const m3u8DataRow$ = "#EXTINF:{0},{1}" & vbCrLf & "{2}"
             With Element
                 Dim f As SFile = __file.IfNullOrEmpty(.File)
-                Dim __f$ = SymbolsConverter.ASCII.EncodeSymbolsOnly(If(Mode = M3U8CreationMode.Absolute, f.ToString, f.File))
+                Dim fStr$ = f.ToString.StringReplaceSymbols({"\"}, "/", EDP.ReturnValue)
+                Dim __f$ = SymbolsConverter.ASCII.Extended.EncodeSymbolsOnly(If(Mode = M3U8CreationMode.Absolute, fStr, f.File), M3U8ExcludedSymbols)
                 If Mode = M3U8CreationMode.Absolute Then __f = $"file:///{__f}"
                 Dim fName$ = .Title.IfNullOrEmpty(f.Name)
                 If MyYouTubeSettings.MusicPlaylistCreate_M3U8_AppendNumber And .PlaylistIndex > 0 Then fName = $"{ .PlaylistIndex}. {fName}"
@@ -1022,12 +1025,19 @@ Namespace API.YouTube.Objects
                     End If
 
                     Dim cDown As Boolean = False
+                    Dim fCover As SFile = Nothing
+                    Dim cUrl$ = String.Empty
                     For Each elem In Elements
                         With DirectCast(elem, YouTubeMediaContainerBase)
-                            If Not .CoverDownloaded Then .CoverDownloaded = cDown
+                            'If Not .CoverDownloaded Then .CoverDownloaded = cDown
+                            .CoverDownloaded = cDown
+                            .CoverFile = fCover
+                            .CoverURL = cUrl
                             AddHandler .FileDownloadStarted, fDown
                             .Download(UseCookies, Token)
                             cDown = .CoverDownloaded
+                            fCover = .CoverFile
+                            cUrl = .CoverURL
                             RemoveHandler .FileDownloadStarted, fDown
                         End With
                         If Token.IsCancellationRequested Or disposedValue Then Exit For
@@ -1054,6 +1064,8 @@ Namespace API.YouTube.Objects
             End Try
         End Sub
         Protected CoverDownloaded As Boolean = False
+        Protected CoverFile As SFile = Nothing
+        Protected CoverURL As String = String.Empty
         Private Sub DownloadPlaylistCover(ByVal PlsId As String, ByVal f As SFile, ByVal UseCookies As Boolean)
             Try
                 Dim url$ = $"https://{IIf(IsMusic, "music", "www")}.youtube.com/playlist?list={PlsId}"
@@ -1089,7 +1101,8 @@ Namespace API.YouTube.Objects
                                 url = LinkFormatterSecure(u)
                                 f.Name = "cover"
                                 f.Extension = "jpg"
-                                If resp.DownloadFile(url, f, EDP.ReturnValue) And f.Exists Then CoverDownloaded = True : AddFile(f)
+                                If resp.DownloadFile(url, f, EDP.ReturnValue) And f.Exists Then _
+                                   CoverFile = f : CoverURL = url : CoverDownloaded = True : AddFile(f)
                             End If
                         End If
                     End Using
@@ -1270,10 +1283,10 @@ Namespace API.YouTube.Objects
                                     End Sub
                                 Dim embedThumbTo As Action(Of SFile) =
                                     Sub(ByVal dFile As SFile)
-                                        If dFile.Exists And ThumbnailFile.Exists Then
+                                        If dFile.Exists And CoverFile.IfNullOrEmpty(ThumbnailFile).Exists Then
                                             Dim dFileNew As SFile = dFile
                                             dFileNew.Name &= "_NEW"
-                                            .Execute($"ffmpeg -i ""{dFile}"" -i ""{ThumbnailFile}"" -map 0:0 -map 1:0 -c copy -id3v2_version 3 -metadata:s:v title=""Cover"" -metadata:s:v comment=""Cover"" ""{dFileNew}""")
+                                            .Execute($"ffmpeg -i ""{dFile}"" -i ""{CoverFile.IfNullOrEmpty(ThumbnailFile)}"" -map 0:0 -map 1:0 -c copy -id3v2_version 3 -metadata:s:v title=""Cover"" -metadata:s:v comment=""Cover"" ""{dFileNew}""")
                                             If dFileNew.Exists AndAlso dFile.Delete(,, EDP.ReturnValue) Then SFile.Rename(dFileNew, dFile)
                                         End If
                                     End Sub
@@ -1353,6 +1366,10 @@ Namespace API.YouTube.Objects
                                     End If
                                 End If
 
+                                'mp3
+                                If IsMusic And ObjectType = YouTubeMediaType.Single And File.Extension = mp3 And
+                                   Not mp3ThumbEmbedded And CoverFile.Exists And MyYouTubeSettings.DefaultAudioEmbedThumbnail_Cover Then embedThumbTo.Invoke(File)
+
                                 'Update video
                                 ThrowAny(Token)
                                 If SelectedVideoIndex >= 0 AndAlso tempFilesList.Count > 0 AndAlso tempFilesList.Exists(Function(tf) tf.ToReplace) Then
@@ -1725,6 +1742,7 @@ Namespace API.YouTube.Objects
                 Dim obj As MediaObject
                 Dim nValue#
                 Dim sValue$
+                Dim allowWebm As Boolean = MyYouTubeSettings.DefaultVideoAllowWebm
                 Dim validCodecValue As Func(Of String, Boolean) = Function(codec) Not codec.IsEmptyString AndAlso Not codec = "none"
 
                 For Each ee In e({"formats"})
@@ -1775,12 +1793,13 @@ Namespace API.YouTube.Objects
                             Dim d As MediaObject = Nothing
                             Dim expWebm As Predicate(Of MediaObject) = Function(mo) mo.Extension = webm
                             Dim expAVC As Predicate(Of MediaObject) = Function(mo) mo.Codec.IfNullOrEmpty("/").ToLower.StartsWith(avc)
-                            Dim comp As Func(Of MediaObject, Predicate(Of MediaObject), Boolean, Boolean) =
-                                Function(mo, exp, isTrue) mo.Type = t And exp.Invoke(mo) = isTrue And mo.Width = d.Width
-                            Dim CountWebm As Func(Of MediaObject, Boolean) = Function(mo) comp.Invoke(mo, expWebm, False)
-                            Dim RemoveWebm As Predicate(Of MediaObject) = Function(mo) comp.Invoke(mo, expWebm, True)
-                            Dim CountAVC As Func(Of MediaObject, Boolean) = Function(mo) comp.Invoke(mo, expAVC, True)
-                            Dim RemoveAVC As Predicate(Of MediaObject) = Function(mo) comp.Invoke(mo, expAVC, False)
+                            Dim comp As Func(Of MediaObject, Predicate(Of MediaObject), Boolean, Boolean, Boolean) =
+                                Function(mo, exp, isTrue, checkHttp) mo.Type = t And exp.Invoke(mo) = isTrue And mo.Width = d.Width And
+                                                                     (Not checkHttp OrElse mo.ProtocolType = Protocols.https)
+                            Dim CountWebm As Func(Of MediaObject, Boolean) = Function(mo) comp.Invoke(mo, expWebm, False, allowWebm)
+                            Dim RemoveWebm As Predicate(Of MediaObject) = Function(mo) comp.Invoke(mo, expWebm, True, allowWebm)
+                            Dim CountAVC As Func(Of MediaObject, Boolean) = Function(mo) comp.Invoke(mo, expAVC, True, False)
+                            Dim RemoveAVC As Predicate(Of MediaObject) = Function(mo) comp.Invoke(mo, expAVC, False, False)
                             For Each d In data
                                 If MediaObjects.Count = 0 Then Exit For
                                 If MediaObjects.LongCount(CountWebm) > 0 Then MediaObjects.RemoveAll(RemoveWebm)

SCrawler.YouTubeDownloader/My Project/AssemblyInfo.vb

@@ -32,6 +32,6 @@ Imports System.Runtime.InteropServices
 ' by using the '*' as shown below:
 ' <Assembly: AssemblyVersion("1.0.*")>
 
-<Assembly: AssemblyVersion("2024.8.10.0")>
-<Assembly: AssemblyFileVersion("2024.8.10.0")>
+<Assembly: AssemblyVersion("2024.10.24.0")>
+<Assembly: AssemblyFileVersion("2024.10.24.0")>
 <Assembly: NeutralResourcesLanguage("en")>

SCrawler/API/Base/DeclaredNames.vb

@@ -28,6 +28,8 @@ Namespace API.Base
         Friend Const GifsDownloadCaption As String = "Download GIFs"
         Friend Const UseMD5ComparisonCaption As String = "Use MD5 comparison"
         Friend Const UseMD5ComparisonToolTip As String = "Each image will be checked for existence using MD5"
+        Friend Const UserNameChangeCaption As String = "UserName"
+        Friend Const UserNameChangeToolTip As String = "If the user has changed their UserName, you can set a new name here. Not required for new users."
         Private Sub New()
         End Sub
     End Class

SCrawler/API/Base/UserDataBase.vb

@@ -947,6 +947,9 @@ BlockNullPicture:
                 LogError(ex, "user information loading error")
             End Try
         End Sub
+        Private Sub UpdateUserInformation_Ex()
+            If _ForceSaveUserInfoOnException Then UpdateUserInformation()
+        End Sub
         Friend Overridable Overloads Sub UpdateUserInformation() Implements IUserData.UpdateUserInformation
             UpdateUserInformation(False)
         End Sub
@@ -1117,6 +1120,7 @@ BlockNullPicture:
         Protected UseClientTokens As Boolean = False
         Protected _ForceSaveUserData As Boolean = False
         Protected _ForceSaveUserInfo As Boolean = False
+        Protected _ForceSaveUserInfoOnException As Boolean = False
         Private _DownloadInProgress As Boolean = False
         Private _EnvirUserExists As Boolean
         Private _EnvirUserSuspended As Boolean
@@ -1136,6 +1140,7 @@ BlockNullPicture:
             _DescriptionEveryTime = Settings.UpdateUserDescriptionEveryTime
             _ForceSaveUserData = False
             _ForceSaveUserInfo = False
+            _ForceSaveUserInfoOnException = False
             _EnvirUserExists = UserExists
             _EnvirUserSuspended = UserSuspended
             _EnvirCreatedByChannel = CreatedByChannel
@@ -1265,9 +1270,11 @@ BlockNullPicture:
                 ThrowIfDisposed()
                 If Not _PictureExists Or _EnvirInvokeUserUpdated Then OnUserUpdated()
             Catch oex As OperationCanceledException When Token.IsCancellationRequested Or TokenPersonal.IsCancellationRequested Or TokenQueue.IsCancellationRequested
+                UpdateUserInformation_Ex()
                 MyMainLOG = $"{ToStringForLog()}: downloading canceled"
                 Canceled = True
             Catch exit_ex As ExitException
+                UpdateUserInformation_Ex()
                 If Not exit_ex.Silent Then
                     If exit_ex.SimpleLogLine Then
                         MyMainLOG = $"{ToStringForLog()}: downloading interrupted (exit) ({exit_ex.Message})"
@@ -1279,6 +1286,7 @@ BlockNullPicture:
             Catch dex As ObjectDisposedException When Disposed
                 Canceled = True
             Catch ex As Exception
+                UpdateUserInformation_Ex()
                 LogError(ex, "downloading data error")
                 HasError = True
             Finally
@@ -1912,6 +1920,7 @@ BlockNullPicture:
                         If m.Contains(IUserData.EraseMode.History) Then
                             If MyFilePosts.Delete(SFO.File, SFODelete.DeleteToRecycleBin, e) Then result = True
                             If MyFileData.Delete(SFO.File, SFODelete.DeleteToRecycleBin, e) Then result = True
+                            If MyMD5File.Delete(SFO.File, SFODelete.DeleteToRecycleBin, e) Then result = True
                             LastUpdated = Nothing
                             EraseData_AdditionalDataFiles()
                             UpdateUserInformation()
@@ -1928,6 +1937,8 @@ BlockNullPicture:
                             _TempMediaList.Clear()
                             _ContentNew.Clear()
                             _ContentList.Clear()
+                            _MD5List.Clear()
+                            _MD5Loaded = False
                         End If
                     End If
                 End If

SCrawler/API/Instagram/EditorExchangeOptions.vb

@@ -7,6 +7,7 @@
 ' This program is distributed in the hope that it will be useful,
 ' but WITHOUT ANY WARRANTY
 Imports SCrawler.Plugin.Attributes
+Imports DN = SCrawler.API.Base.DeclaredNames
 Namespace API.Instagram
     Friend Class EditorExchangeOptions
 #Region "Download"
@@ -35,6 +36,8 @@ Namespace API.Instagram
 #End Region
         <PSetting(Caption:="Place the extracted image into the video folder")>
         Friend Property PutImageVideoFolder As Boolean
+        <PSetting(Address:=SettingAddress.User, Caption:=DN.UserNameChangeCaption, ToolTip:=DN.UserNameChangeToolTip)>
+        Friend Overridable Property UserName As String = String.Empty
         Friend Sub New(ByVal u As UserData)
             With u
                 GetTimeline = .GetTimeline
@@ -50,6 +53,8 @@ Namespace API.Instagram
                 GetTagged_VideoPic = .GetTaggedData_VideoPic
 
                 PutImageVideoFolder = .PutImageVideoFolder
+
+                UserName = .NameTrue(True)
             End With
         End Sub
         Friend Sub New(ByVal s As SiteSettings)

SCrawler/API/Instagram/UserData.vb

@@ -112,9 +112,9 @@ Namespace API.Instagram
             End Select
         End Function
         Protected _NameTrue As String = String.Empty
-        Friend ReadOnly Property NameTrue As String
+        Friend ReadOnly Property NameTrue(Optional ByVal Exact As Boolean = False) As String
             Get
-                Return _NameTrue.IfNullOrEmpty(Name)
+                Return If(Exact, _NameTrue, _NameTrue.IfNullOrEmpty(Name))
             End Get
         End Property
         Private UserNameRequested As Boolean = False
@@ -178,6 +178,8 @@ Namespace API.Instagram
                     GetTaggedData_VideoPic = .GetTagged_VideoPic
 
                     PutImageVideoFolder = .PutImageVideoFolder
+
+                    _NameTrue = .UserName
                 End With
             End If
         End Sub
@@ -631,7 +633,7 @@ Namespace API.Instagram
                         'Check environment
                         If Not IsSavedPosts Then
                             If ID.IsEmptyString Then GetUserData()
-                            If ID.IsEmptyString Then Throw New Plugin.ExitException("can't get user ID")
+                            If ID.IsEmptyString Then UserExists = False : _ForceSaveUserInfoOnException = True : Throw New Plugin.ExitException("can't get user ID")
                             If _UseGQL And Cursor.IsEmptyString And Not Section = Sections.SavedPosts Then
                                 If Not ValidateBaseTokens() Then GetPageTokens()
                                 If Not ValidateBaseTokens(TokensErrData) Then ValidateBaseTokens_Error(TokensErrData)
@@ -1171,6 +1173,7 @@ NextPageBlock:
                     End Using
                 End If
             Catch ex As Exception
+                UserExists = False
                 If Not __idFound Then
                     If Responser.StatusCode = HttpStatusCode.NotFound Or Responser.StatusCode = HttpStatusCode.BadRequest Then
                         Throw ex