Merge pull request #53 from hmlendea/dev

Improved source checking, data tweaks, fixes
hmlendea · Oct 18, 2020 · 3d0c696 · 3d0c696
2 parents e42d5d0 + 3ab6ac9
commit 3d0c696
Show file tree

Hide file tree

Showing 14 changed files with 209 additions and 61 deletions.
diff --git a/Configuration/ApplicationSettings.cs b/Configuration/ApplicationSettings.cs
@@ -10,6 +10,8 @@ public sealed class ApplicationSettings
 
         public bool AreTvGuideTagsEnabled { get; set; }
 
+        public bool ArePlaylistDetailsTagsEnabled { get; set; }
+
         public string UserAgent { get; set; }
     }
 }
diff --git a/Data/channels.xml b/Data/channels.xml
@@ -35,9 +35,10 @@
     <GroupId>music</GroupId>
     <LogoUrl>https://raw.githubusercontent.com/hmlendea/tv-logos/master/logos/1_music_channel.png</LogoUrl>
     <Aliases>
-      <string>RO: Music Channel</string>
       <string>RO: 1 Music Channele</string>
+      <string>RO: 1 Music</string>
       <string>RO: Music 1 Channel</string>
+      <string>RO: Music Channel</string>
     </Aliases>
   </ChannelDefinitionEntity>
   <ChannelDefinitionEntity>
@@ -183,7 +184,7 @@
     <Name>AMC</Name>
     <Country>RO</Country>
     <GroupId>movies</GroupId>
-    <LogoUrl>http://storage.alltvplay.com/images/channels/57d2bd46bc33b1566d8b4694.png</LogoUrl>
+    <LogoUrl>https://raw.githubusercontent.com/hmlendea/tv-logos/master/logos/amc.png</LogoUrl>
   </ChannelDefinitionEntity>
   <ChannelDefinitionEntity>
     <Id>AnimalPlanet.ro</Id>
@@ -770,6 +771,17 @@
       <string>DW</string>
     </Aliases>
   </ChannelDefinitionEntity>
+  <ChannelDefinitionEntity>
+    <Id>DiasporaMedia.fr</Id>
+    <Name>Diaspora Media</Name>
+    <Country>MD</Country>
+    <GroupId>music</GroupId>
+    <LogoUrl>https://raw.githubusercontent.com/hmlendea/tv-logos/master/logos/diaspora_media.png</LogoUrl>
+    <Aliases>
+      <string>MD: Diaspora Media</string>
+      <string>RO: Diaspora Media</string>
+    </Aliases>
+  </ChannelDefinitionEntity>
   <ChannelDefinitionEntity>
     <Id>DiasporaTV.ro</Id>
     <Name>Diaspora TV</Name>
@@ -1216,6 +1228,17 @@
       <string>RO: Extreme Sports Channel</string>
     </Aliases>
   </ChannelDefinitionEntity>
+  <ChannelDefinitionEntity>
+    <Id>FagarasTV.ro</Id>
+    <Name>Făgăraș TV</Name>
+    <Country>RO</Country>
+    <GroupId>local-transilvania</GroupId>
+    <LogoUrl>https://raw.githubusercontent.com/hmlendea/radio-logos/master/logos/fagaras_tv.png</LogoUrl>
+    <Aliases>
+      <string>RO: Făgăraș</string>
+      <string>RO: TV Făgăraș</string>
+    </Aliases>
+  </ChannelDefinitionEntity>
   <ChannelDefinitionEntity>
     <Id>Fallout76AppalachiaRadio.com</Id>
     <Name>Fallout 76 Appalachia Radio</Name>
@@ -1762,6 +1785,7 @@
     <Aliases>
       <string>-----ROMANIA------</string>
       <string>iptvcat.com</string>
+      <string>RO: iptvcat.com</string>
     </Aliases>
   </ChannelDefinitionEntity>
   <ChannelDefinitionEntity>
@@ -1859,6 +1883,17 @@
       <string>RO: Linkpress</string>
     </Aliases>
   </ChannelDefinitionEntity>
+  <ChannelDefinitionEntity>
+    <Id>LiraTV.es</Id>
+    <Name>Lira TV</Name>
+    <Country>ES</Country>
+    <GroupId>international</GroupId>
+    <LogoUrl>https://raw.githubusercontent.com/hmlendea/tv-logos/master/logos/lira_tv.png</LogoUrl>
+    <Aliases>
+      <string>RO: Lira TV</string>
+      <string>RO: Lira</string>
+    </Aliases>
+  </ChannelDefinitionEntity>
   <ChannelDefinitionEntity>
     <Id>LitoralTV.ro</Id>
     <Name>Litoral TV</Name>
@@ -2205,7 +2240,7 @@
     <Name>Misa Senzațional TV</Name>
     <Country>RO</Country>
     <GroupId>religious</GroupId>
-    <LogoUrl>https://misatv.ro/wp-content/uploads/2018/09/MTV-Logo-Cropped2.png</LogoUrl>
+    <LogoUrl>https://raw.githubusercontent.com/hmlendea/tv-logos/master/logos/misa_senzational.png</LogoUrl>
     <Aliases>
       <string>RO: Misa</string>
     </Aliases>
@@ -2309,6 +2344,7 @@
     <Name>MTV Live</Name>
     <Country>RO</Country>
     <GroupId>music</GroupId>
+    <LogoUrl>https://raw.githubusercontent.com/hmlendea/tv-logos/master/logos/mtv_live.png</LogoUrl>
   </ChannelDefinitionEntity>
   <ChannelDefinitionEntity>
     <Id>MTVMusic.ro</Id>
@@ -2422,6 +2458,7 @@
     <Aliases>
       <string>Naional TV</string>
       <string>RO: Na?ional</string>
+      <string>RO: Național</string>
       <string>RO: TV Național</string>
     </Aliases>
   </ChannelDefinitionEntity>
@@ -3437,14 +3474,24 @@
     <Country>RO</Country>
     <GroupId>unknown</GroupId>
   </ChannelDefinitionEntity>
+  <ChannelDefinitionEntity>
+    <Id>TVArad.ro</Id>
+    <Name>TV Arad</Name>
+    <Country>RO</Country>
+    <GroupId>local-crisana</GroupId>
+    <LogoUrl>https://raw.githubusercontent.com/hmlendea/tv-logos/master/logos/tv_arad.png</LogoUrl>
+    <Aliases>
+      <string>RO: Arad TV</string>
+    </Aliases>
+  </ChannelDefinitionEntity>
   <ChannelDefinitionEntity>
     <Id>TVBuzau.ro</Id>
     <Name>TV Buzău</Name>
     <Country>RO</Country>
     <GroupId>local-muntenia</GroupId>
     <LogoUrl>http://stvron.com/canale/mari/tv-buzau.png</LogoUrl>
     <Aliases>
-      <string>Buzău TV</string>
+      <string>RO: Buzău TV</string>
     </Aliases>
   </ChannelDefinitionEntity>
   <ChannelDefinitionEntity>

diff --git a/Data/providers.xml b/Data/providers.xml
@@ -18,7 +18,7 @@
     <AllowCaching>false</AllowCaching>
     <Name>hmlendea's GitHub Channels MD</Name>
     <UrlFormat>https://raw.githubusercontent.com/hmlendea/iptv-playlist/master/md.m3u</UrlFormat>
-    <Country>M</Country>
+    <Country>MD</Country>
   </PlaylistProviderEntity>
 
   <PlaylistProviderEntity>
@@ -76,16 +76,6 @@
     <Country>MD</Country>
   </PlaylistProviderEntity>
 
-  <PlaylistProviderEntity>
-    <Id>github-cstoicescu-ro</Id>
-    <IsEnabled>true</IsEnabled>
-    <Priority>120</Priority>
-    <Name>cstoicescu's GitHub RO Channels</Name>
-    <AllowCaching>false</AllowCaching>
-    <UrlFormat>https://raw.githubusercontent.com/cstoicescu/iptv-free-tv/master/playlist.m3u</UrlFormat>
-    <Country>RO</Country>
-  </PlaylistProviderEntity>
-
   <PlaylistProviderEntity>
     <Id>github-makiptv-ro</Id>
     <IsEnabled>true</IsEnabled>
@@ -727,24 +717,6 @@
     <Country>RO</Country>
   </PlaylistProviderEntity>
 
-  <PlaylistProviderEntity>
-    <Id>github-vb6rocod-ro</Id>
-    <IsEnabled>true</IsEnabled>
-    <Priority>960</Priority>
-    <Name>vb6rocod's IPTV GitHub RO Channels</Name>
-    <UrlFormat>https://raw.githubusercontent.com/vb6rocod/hddlinks_android/master/tv/pl/Frecvente.m3u</UrlFormat>
-    <Country>RO</Country>
-  </PlaylistProviderEntity>
-
-  <PlaylistProviderEntity>
-    <Id>github-vb6rocod-ro-tvr</Id>
-    <IsEnabled>true</IsEnabled>
-    <Priority>961</Priority>
-    <Name>vb6rocod's IPTV GitHub RO TVR Channels</Name>
-    <UrlFormat>https://raw.githubusercontent.com/vb6rocod/hddlinks_android/master/tv/pl/TVR.m3u</UrlFormat>
-    <Country>RO</Country>
-  </PlaylistProviderEntity>
-
   <PlaylistProviderEntity>
     <Id>github-SharyRajpoot-ro</Id>
     <IsEnabled>true</IsEnabled>
@@ -828,6 +800,34 @@
     <Country>RO</Country>
   </PlaylistProviderEntity>
 
+  <PlaylistProviderEntity>
+    <Id>github-cstoicescu-ro</Id>
+    <IsEnabled>false</IsEnabled>
+    <Priority>1000</Priority>
+    <Name>cstoicescu's GitHub RO Channels</Name>
+    <AllowCaching>false</AllowCaching>
+    <UrlFormat>https://raw.githubusercontent.com/cstoicescu/iptv-free-tv/master/playlist.m3u</UrlFormat>
+    <Country>RO</Country>
+  </PlaylistProviderEntity>
+
+  <PlaylistProviderEntity>
+    <Id>github-vb6rocod-ro</Id>
+    <IsEnabled>true</IsEnabled>
+    <Priority>1100</Priority>
+    <Name>vb6rocod's IPTV GitHub RO Channels</Name>
+    <UrlFormat>https://raw.githubusercontent.com/vb6rocod/hddlinks_android/master/tv/pl/Frecvente.m3u</UrlFormat>
+    <Country>RO</Country>
+  </PlaylistProviderEntity>
+
+  <PlaylistProviderEntity>
+    <Id>github-vb6rocod-ro-tvr</Id>
+    <IsEnabled>true</IsEnabled>
+    <Priority>1101</Priority>
+    <Name>vb6rocod's IPTV GitHub RO TVR Channels</Name>
+    <UrlFormat>https://raw.githubusercontent.com/vb6rocod/hddlinks_android/master/tv/pl/TVR.m3u</UrlFormat>
+    <Country>RO</Country>
+  </PlaylistProviderEntity>
+
   <PlaylistProviderEntity>
     <Id>github-hmlendea-ro-timed</Id>
     <IsEnabled>true</IsEnabled>

diff --git a/README.md b/README.md
@@ -91,6 +91,7 @@ The settings are stored in the `appsettings.json` file in the root directory.
  - *daysToCheck*: How far in the past to go for each playlist. If today's playlist is not found (sometimes the providers skip some days) then the service will move on to the previous day, again and again until one is found or the daysToCheck limit is reached.
  - *canIncludeUnmatchedChannels*: Boolean value indicating whether provider channels that were not able to be matched with the data in the channelStorePath file should be included in the output file (in the Unknown category) or not at all
  - *areTvGuideTagsEnabled*: Boolean value indicating whether TV Guide tags (logo URLs, groups, TVG IDs, channel numbers, etc) should be included in the output file or not
+ - *arePlaylistDetailsTagsEnabled*: Boolean value indicating whether playlist details (playlist ID, the playlist's original channel name) should be included in the output file or not
  - *userAgent*: String value indicating the UserAgent that should be used when performing HTTP operations
 
 ### Channel data

diff --git a/Service/ChannelMatcher.cs b/Service/ChannelMatcher.cs
@@ -21,8 +21,8 @@ public sealed class ChannelMatcher : IChannelMatcher
             { "\\[B\\]", "" },
 
             { "[\\(\\[]]*([Aa]uto|[Bb]ackup|[Ll]ive [Oo]n [Mm]atches|[Mm]ulti-*[Aa]udio|[Mm]ulti-*[Ss]ub|[Nn]ew!*|[Oo]n-[Dd]emand)[\\)\\]]*", "" },
-            { "([a-zA-Z0-9_ ]{3})[ _\\|\\[\\(\\]\\)\".:-](Ultra|[FU])*_*[HMS][DQ]", "$1" },
-            { "4[Kk]\\+*", "" },
+            { "([a-zA-Z0-9_ ]{4})[ _\\|\\[\\(\\]\\)\".:-](Ultra|[FU])*_*[HMS][DQ]", "$1" },
+            { "4[Kk]\\+", "" },
 
             { "^( *[\\|\\[\\(\\]\\)\".:-]* *([A-Z][A-Z]) *[\\|\\[\\(\\]\\)\".:-] *)+", "$2:" },
             { "^ *([A-Z][A-Z]): *(.*) \\(*\\1\\)*$", "$1: $2" },
@@ -67,27 +67,29 @@ public ChannelMatcher(ICacheManager cache)
 
         public string NormaliseName(string name, string country)
         {
-            string normalisedName = cache.GetNormalisedChannelName(name);
-
-            if (!string.IsNullOrWhiteSpace(normalisedName))
-            {
-                return normalisedName;
-            }
+            string fullName;
 
             if (string.IsNullOrWhiteSpace(country))
             {
-                normalisedName = name;
+                fullName = name;
             }
             else
             {
-                normalisedName = $"{country}: {name}";
+                fullName = $"{country}: {name}";
+            }
+
+            string normalisedName = cache.GetNormalisedChannelName(fullName);
+
+            if (!string.IsNullOrWhiteSpace(normalisedName))
+            {
+                return normalisedName;
             }
 
-            normalisedName = normalisedName.RemoveDiacritics();
+            normalisedName = fullName.RemoveDiacritics();
             normalisedName = StripChannelName(normalisedName);
             normalisedName = normalisedName.ToUpper();
 
-            cache.StoreNormalisedChannelName(name, normalisedName);
+            cache.StoreNormalisedChannelName(fullName, normalisedName);
 
             return normalisedName;
         }

diff --git a/Service/MediaSourceChecker.cs b/Service/MediaSourceChecker.cs
@@ -1,5 +1,6 @@
 using System;
 using System.IO;
+using System.Linq;
 using System.Net;
 using System.Text;
 using System.Text.RegularExpressions;
@@ -16,8 +17,14 @@ namespace IptvPlaylistAggregator.Service
     public sealed class MediaSourceChecker : IMediaSourceChecker
     {
         const string YouTubeVideoUrlPattern = "^(https?\\:\\/\\/)?(www\\.youtube\\.com|youtu\\.?be)\\/.+$";
+        const string TinyUrlPattern = "^(https?\\:\\/\\/)?((www\\.)?tinyurl\\.com)\\/.+$";
         const string NonHttpUrlPattern = "^(?!http).*";
 
+        static readonly string[] BlacklistedSources = new string[]
+        {
+            "http://hls.protv.md/acasatv/acasatv.m3u8"
+        };
+
         readonly IFileDownloader fileDownloader;
         readonly IPlaylistFileBuilder playlistFileBuilder;
         readonly ICacheManager cache;
@@ -55,6 +62,10 @@ public async Task<bool> IsSourcePlayableAsync(string url)
             {
                 state = StreamState.Unsupported;
             }
+            else if (IsUrlBlacklisted(url))
+            {
+                state = StreamState.Blacklisted;
+            }
             else if (url.Contains(".m3u") || url.Contains(".m3u8"))
             {
                 state = await GetPlaylistStateAsync(url);
@@ -80,11 +91,27 @@ public async Task<bool> IsSourcePlayableAsync(string url)
         bool IsUrlUnsupported(string url)
         {
             if (Regex.IsMatch(url, YouTubeVideoUrlPattern) ||
+                Regex.IsMatch(url, TinyUrlPattern) ||
                 Regex.IsMatch(url, NonHttpUrlPattern))
             {
                 return true;
             }
 
+            if (url.EndsWith(".mp4") || url.Contains(".mp4?"))
+            {
+                return true;
+            }
+
+            return false;
+        }
+
+        bool IsUrlBlacklisted(string url)
+        {
+            if (BlacklistedSources.Any(x => x.Contains(url)))
+            {
+                return true;
+            }
+
             return false;
         }
 
@@ -104,10 +131,26 @@ async Task<StreamState> GetPlaylistStateAsync(string url)
             {
                 return StreamState.Dead;
             }
-            else
+
+            foreach (Channel channel in playlist.Channels)
             {
-                return StreamState.Alive;
+                string channelUrl = channel.Url;
+
+                if (!channelUrl.StartsWith("http"))
+                {
+                    // TODO: Replace this with something proper
+                    channelUrl = Path.GetDirectoryName(url).Replace(":/", "://") + "/" + channelUrl;
+                }
+
+                bool isPlayable = await IsSourcePlayableAsync(channelUrl);
+
+                if (isPlayable)
+                {
+                    return StreamState.Alive;
+                }
             }
+
+            return StreamState.Dead;
         }
 
         async Task<StreamState> GetStreamStateAsync(string url)
@@ -189,7 +232,7 @@ async Task<HttpStatusCode> GetHttpStatusCode(string url)
 
         HttpWebRequest CreateWebRequest(string url)
         {
-            const int timeout = 3000;
+            const int timeout = 10000;
 
             HttpWebRequest request = (HttpWebRequest)HttpWebRequest.Create(url);
             request.Method = "GET";

diff --git a/Service/Models/Channel.cs b/Service/Models/Channel.cs
@@ -14,6 +14,10 @@ public sealed class Channel
 
         public int Number { get; set; }
 
+        public string PlaylistId { get; set; }
+
+        public string PlaylistChannelName { get; set; }
+
         public string Url { get; set; }
     }
 }