diff --git a/GithubNet/GithubNet.csproj b/GithubNet/GithubNet.csproj index be45727..253b741 100644 --- a/GithubNet/GithubNet.csproj +++ b/GithubNet/GithubNet.csproj @@ -6,7 +6,7 @@ enable True GithubNet - 0.1 + 0.2 liebki This library allows you to retrieve trending GitHub repositories and their information. liebki @@ -18,6 +18,7 @@ True icon.png GithubDataNet + The update includes a new GetRepositoryInfoAsync() method for retrieving detailed information about GitHub repositories, a 'Repository' class to support it, and improved data types (TotalStars and TotalForks changed to integers). diff --git a/GithubNet/GithubNetClient.cs b/GithubNet/GithubNetClient.cs index 0ccbc94..a161acb 100644 --- a/GithubNet/GithubNetClient.cs +++ b/GithubNet/GithubNetClient.cs @@ -1,4 +1,6 @@ -namespace GithubNet +using GithubNet.Models; + +namespace GithubNet { public class GithubNetClient { @@ -14,6 +16,18 @@ public async Task> GetTrendItemsAsync(bool loadTrendItemDetails } } + public async Task GetRepositoryInfoAsync(string repositoryLink) + { + if (!string.IsNullOrEmpty(repositoryLink) && repositoryLink.StartsWith("https://github.com/", StringComparison.InvariantCultureIgnoreCase)) + { + return await GithubNetManager.GetRepositoryInfo(repositoryLink); + } + else + { + throw new ArgumentException("The parameter customquery, can't be whitespace, null or empty also it has to contain some kind of https://github.com/trending-url string!"); + } + } + public async Task GetTrendItemDetailsAsync(TrendItem entryItem) { if (entryItem != null && !string.IsNullOrEmpty(entryItem.RespositoryLink) && !string.IsNullOrWhiteSpace(entryItem.RespositoryLink)) diff --git a/GithubNet/GithubNetManager.cs b/GithubNet/GithubNetManager.cs index ff65c58..345cc54 100644 --- a/GithubNet/GithubNetManager.cs +++ b/GithubNet/GithubNetManager.cs @@ -1,6 +1,7 @@ -using HtmlAgilityPack; -using System.Net; +using System.Net; using System.Text; +using HtmlAgilityPack; +using GithubNet.Models; namespace GithubNet { @@ -32,7 +33,7 @@ internal static async Task> GetAllTrendEntries(bool loadTrendIte } string UsernameFiltered = FilterLineBreaks(Username.InnerText.Replace(" /", string.Empty)); - string RepositoryLinkFiltered = $"https://github.com{RepositoryLink.Attributes["href"].Value}"; + string RepositoryLinkFiltered = $"https://github.com{RepositoryLink.GetAttributeValue("href", string.Empty)}"; string[] RepositoryNameParse = RepositoryLinkFiltered.Split('/'); string RepositoryName = "An error occured"; @@ -42,8 +43,11 @@ internal static async Task> GetAllTrendEntries(bool loadTrendIte RepositoryName = RepositoryNameParse[4]; } - string TotalStarsFiltered = FilterLineBreaks(TotalForksAndStars[0].InnerText); - string TotalForksFiltered = FilterLineBreaks(TotalForksAndStars[1].InnerText); + string TotalStars = FilterLineBreaks(TotalForksAndStars[0].InnerText); + int.TryParse(TotalStars.Replace(",", string.Empty), out int TotalStarsFiltered); + + string TotalForks = FilterLineBreaks(TotalForksAndStars[1].InnerText); + int.TryParse(TotalForks.Replace(",", string.Empty), out int TotalForksFiltered); string ProgrammingLanguage = "None"; if (productElement.QuerySelectorAll("article.Box-row > div > span > span") != null) @@ -57,12 +61,12 @@ internal static async Task> GetAllTrendEntries(bool loadTrendIte if (loadTrendItemDetails) { - TrendItem trendItem = new(UsernameFiltered, RepositoryLinkFiltered, RepositoryName, DescriptionFiltered, TotalStarsFiltered, TotalForksFiltered, ProgrammingLanguage); + TrendItem trendItem = new(ProgrammingLanguage, false, false, string.Empty, string.Empty, UsernameFiltered, RepositoryLinkFiltered, RepositoryName, DescriptionFiltered, TotalStarsFiltered, TotalForksFiltered, false, string.Empty, false, Array.Empty()); TrendingRepositoriesList.Add(await GetTrendDetails(trendItem)); } else { - TrendingRepositoriesList.Add(new(UsernameFiltered, RepositoryLinkFiltered, RepositoryName, DescriptionFiltered, TotalStarsFiltered, TotalForksFiltered, ProgrammingLanguage)); + TrendingRepositoriesList.Add(new(ProgrammingLanguage, false, false, string.Empty, string.Empty, UsernameFiltered, RepositoryLinkFiltered, RepositoryName, DescriptionFiltered, TotalStarsFiltered, TotalForksFiltered, false, string.Empty, false, Array.Empty())); } } catch (Exception) @@ -74,6 +78,92 @@ internal static async Task> GetAllTrendEntries(bool loadTrendIte return TrendingRepositoriesList; } + private static async Task FillBaseData(HtmlNode node, string repositoryUrl) + { + HtmlNode Username = node.SelectSingleNode("/html/body/div[1]/div[4]/div/main/div/div[1]/div[1]/div/span[1]/a"); + HtmlNode RepositoryName = node.SelectSingleNode("/html/body/div[1]/div[4]/div/main/div/div[1]/div[1]/div/strong/a"); + + HtmlNode Description = node.SelectSingleNode("/html/body/div[1]/div[4]/div/main/turbo-frame/div/div/div/div[2]/div[2]/div/div[1]/div/p"); + HtmlNode TotalForks = node.SelectSingleNode("//*[@id=\"repo-network-counter\"]"); + + HtmlNode TotalStars = node.SelectSingleNode("//*[@id=\"repo-stars-counter-star\"]"); + HtmlNode ProjectUrl = node.SelectSingleNode("/html/body/div[1]/div[4]/div/main/turbo-frame/div/div/div/div[2]/div[2]/div/div[1]/div/div[1]/span/a"); + + HtmlNode Topics = node.SelectSingleNode("/html/body/div[1]/div[4]/div/main/turbo-frame/div/div/div/div[2]/div[2]/div/div[1]/div/div[2]/div"); + + string DescriptionFiltered = FilterLineBreaks(Description.InnerText); + if (!string.IsNullOrWhiteSpace(DescriptionFiltered)) + { + DescriptionFiltered = WebUtility.HtmlDecode(DescriptionFiltered); + } + + List TopicsFiltered = new(); + bool HasTopics = false; + + if (Topics != null) + { + HasTopics = true; + IList TopicsValues = Topics.QuerySelectorAll("a.topic-tag"); + + foreach (HtmlNode t in TopicsValues) + { + TopicsFiltered.Add(FilterLineBreaks(t.InnerHtml)); + } + } + + string ProjectUrlFiltered = string.Empty; + bool HasProjectUrl = false; + + if (ProjectUrl != null) + { + HasProjectUrl = true; + ProjectUrlFiltered = ProjectUrl.GetAttributeValue("href", string.Empty); + } + + string ForksValue = TotalForks.GetAttributeValue("title", string.Empty); + int.TryParse(ForksValue.Replace(",", string.Empty), out int ForksFiltered); + + string StarsValue = TotalStars.GetAttributeValue("title", string.Empty); + int.TryParse(StarsValue.Replace(",", string.Empty), out int StarsFiltered); + + string UsernameFiltered = FilterLineBreaks(Username.InnerText.Replace(" /", string.Empty)); + string RepositoryNameFiltered = FilterLineBreaks(RepositoryName.InnerText.Replace(" /", string.Empty)); + + return new(UsernameFiltered, repositoryUrl, RepositoryNameFiltered, DescriptionFiltered, StarsFiltered, ForksFiltered, HasProjectUrl, ProjectUrlFiltered, HasTopics, TopicsFiltered.ToArray()); + } + + internal static async Task GetRepositoryInfo(string repositoryLink) + { + HtmlWeb CommitUrl = new(); + HtmlDocument TrendingRep = CommitUrl.Load(repositoryLink); + + ItemBase baseData = await FillBaseData(TrendingRep.DocumentNode, repositoryLink); + HtmlNode OpenIssuesNumber = TrendingRep.DocumentNode.SelectSingleNode("//*[@id=\"issues-repo-tab-count\"]"); + + HtmlNode OpenPullRequestsNumber = TrendingRep.DocumentNode.SelectSingleNode("//*[@id=\"pull-requests-repo-tab-count\"]"); + HtmlNode TotalCommitsNumber = TrendingRep.DocumentNode.SelectSingleNode("/html/body/div[1]/div[4]/div/main/turbo-frame/div/div/div/div[2]/div[1]/div[2]/div[1]/div/div[4]/ul/li/a/span/strong"); + + HtmlNode TotalContributorsNumber = TrendingRep.DocumentNode.SelectSingleNode("/html/body/div[1]/div[4]/div/main/turbo-frame/div/div/div/div[2]/div[2]/div/div[5]/div/h2/a/span"); + + string OpenIssuesNumberFiltered = FilterLineBreaks(OpenIssuesNumber.InnerHtml); + int.TryParse(OpenIssuesNumberFiltered.Replace(",", string.Empty), out int OpenIssuesNumberValue); + + string OpenPullRequestsNumberFiltered = FilterLineBreaks(OpenPullRequestsNumber.InnerHtml); + int.TryParse(OpenPullRequestsNumberFiltered.Replace(",", string.Empty), out int OpenPullRequestsNumberValue); + + string TotalCommitsNumberFiltered = FilterLineBreaks(TotalCommitsNumber.InnerText); + int.TryParse(TotalCommitsNumberFiltered.Replace(",", string.Empty), out int TotalCommitsNumberValue); + + int TotalContributorsNumberValue = 0; + if (TotalContributorsNumber != null) + { + string TotalContributorsNumberFiltered = FilterLineBreaks(TotalContributorsNumber.InnerHtml); + int.TryParse(TotalContributorsNumberFiltered.Replace(",", string.Empty), out TotalContributorsNumberValue); + } + + return new Repository(OpenIssuesNumberValue, OpenPullRequestsNumberValue, TotalCommitsNumberValue, TotalContributorsNumberValue, baseData.User, baseData.RespositoryLink, baseData.RespositoryName, baseData.Description, baseData.TotalStars, baseData.TotalForks, baseData.HasProjectUrl, baseData.ProjectUrl, baseData.HasTopics, baseData.Topics); + } + internal static async Task GetTrendDetails(TrendItem entryItem) { entryItem.HasDetails = true; @@ -93,7 +183,7 @@ internal static async Task GetTrendDetails(TrendItem entryItem) if (ProjectUrl != null) { entryItem.HasProjectUrl = true; - entryItem.ProjectUrl = ProjectUrl.Attributes["href"].Value; + entryItem.ProjectUrl = ProjectUrl.GetAttributeValue("href", string.Empty); } if (Topics.Count > 0) diff --git a/GithubNet/TrendItem.cs b/GithubNet/Models/ItemBase.cs similarity index 51% rename from GithubNet/TrendItem.cs rename to GithubNet/Models/ItemBase.cs index 3860ef2..6f6f247 100644 --- a/GithubNet/TrendItem.cs +++ b/GithubNet/Models/ItemBase.cs @@ -1,12 +1,8 @@ -namespace GithubNet +namespace GithubNet.Models { - public class TrendItem + public class ItemBase { - public TrendItem() - { - } - - public TrendItem(string user, string respositoryLink, string respositoryName, string description, string totalStars, string totalForks, string programminglanguage) + public ItemBase(string user, string respositoryLink, string respositoryName, string description, int totalStars, int totalForks, bool hasProjectUrl, string projectUrl, bool hasTopics, string[] topics) { User = user; RespositoryLink = respositoryLink; @@ -14,7 +10,10 @@ public TrendItem(string user, string respositoryLink, string respositoryName, st Description = description; TotalStars = totalStars; TotalForks = totalForks; - Programminglanguage = programminglanguage; + HasProjectUrl = hasProjectUrl; + ProjectUrl = projectUrl; + HasTopics = hasTopics; + Topics = topics; } public string User { get; set; } @@ -25,15 +24,9 @@ public TrendItem(string user, string respositoryLink, string respositoryName, st public string Description { get; set; } - public string TotalStars { get; set; } - - public string TotalForks { get; set; } - - public string Programminglanguage { get; set; } + public int TotalStars { get; set; } - public bool HasDetails { get; set; } - - public bool IsArchived { get; set; } + public int TotalForks { get; set; } public bool HasProjectUrl { get; set; } @@ -43,9 +36,6 @@ public TrendItem(string user, string respositoryLink, string respositoryName, st public string[] Topics { get; set; } - public string LastCommitTime { get; set; } - - public string LastCommitUrl { get; set; } public string GetLastCommitUrl(string branch) { @@ -71,10 +61,5 @@ public string GetStarsUrl() { return $"https://github.com/{this.User}/{this.RespositoryName}/stargazers"; } - - public override string ToString() - { - return $"{{{nameof(User)}={User}, {nameof(RespositoryLink)}={RespositoryLink}, {nameof(RespositoryName)}={RespositoryName}, {nameof(Description)}={Description}, {nameof(TotalStars)}={TotalStars}, {nameof(TotalForks)}={TotalForks}, {nameof(Programminglanguage)}={Programminglanguage}, {nameof(HasDetails)}={HasDetails.ToString()}, {nameof(IsArchived)}={IsArchived.ToString()}, {nameof(HasProjectUrl)}={HasProjectUrl.ToString()}, {nameof(ProjectUrl)}={ProjectUrl}, {nameof(HasTopics)}={HasTopics.ToString()}, {nameof(Topics)}={Topics}, {nameof(LastCommitTime)}={LastCommitTime}, {nameof(LastCommitUrl)}={LastCommitUrl}}}"; - } } } \ No newline at end of file diff --git a/GithubNet/Models/Repository.cs b/GithubNet/Models/Repository.cs new file mode 100644 index 0000000..40148bd --- /dev/null +++ b/GithubNet/Models/Repository.cs @@ -0,0 +1,38 @@ +namespace GithubNet.Models +{ + public class Repository : ItemBase + { + public Repository(int openIssuesNumber, int openPullRequestsNumber, int totalCommitsNumber, int totalContributorsNumber, string user, string respositoryLink, string respositoryName, string description, int totalStars, int totalForks, bool hasProjectUrl, string projectUrl, bool hasTopics, string[] topics) : base(user, respositoryLink, respositoryName, description, totalStars, totalForks, hasProjectUrl, projectUrl, hasTopics, topics) + { + OpenIssuesNumber = openIssuesNumber; + OpenPullRequestsNumber = openPullRequestsNumber; + TotalCommitsNumber = totalCommitsNumber; + TotalContributorsNumber = totalContributorsNumber; + + base.User = user; + base.RespositoryLink = respositoryLink; + base.RespositoryName = respositoryName; + base.Description = description; + base.TotalStars = totalStars; + base.TotalForks = totalForks; + base.HasProjectUrl = hasProjectUrl; + base.ProjectUrl = projectUrl; + base.HasTopics = hasTopics; + base.Topics = topics; + + } + + public int OpenIssuesNumber { get; set; } + + public int OpenPullRequestsNumber { get; set; } + + public int TotalCommitsNumber { get; set; } + + public int TotalContributorsNumber { get; set; } + + public override string ToString() + { + return $"{{{nameof(OpenIssuesNumber)}={OpenIssuesNumber.ToString()}, {nameof(OpenPullRequestsNumber)}={OpenPullRequestsNumber.ToString()}, {nameof(TotalCommitsNumber)}={TotalCommitsNumber.ToString()}, {nameof(TotalContributorsNumber)}={TotalContributorsNumber.ToString()}, {nameof(User)}={User}, {nameof(RespositoryLink)}={RespositoryLink}, {nameof(RespositoryName)}={RespositoryName}, {nameof(Description)}={Description}, {nameof(TotalStars)}={TotalStars.ToString()}, {nameof(TotalForks)}={TotalForks.ToString()}, {nameof(HasProjectUrl)}={HasProjectUrl.ToString()}, {nameof(ProjectUrl)}={ProjectUrl}, {nameof(HasTopics)}={HasTopics.ToString()}, {nameof(Topics)}={Topics}}}"; + } + } +} \ No newline at end of file diff --git a/GithubNet/Models/TrendItem.cs b/GithubNet/Models/TrendItem.cs new file mode 100644 index 0000000..4fc7be4 --- /dev/null +++ b/GithubNet/Models/TrendItem.cs @@ -0,0 +1,36 @@ +namespace GithubNet.Models +{ + public class TrendItem : ItemBase + { + + public TrendItem(string programminglanguage, bool hasDetails, bool isArchived, string lastCommitTime, string lastCommitUrl, string user, string respositoryLink, string respositoryName, string description, int totalStars, int totalForks, bool hasProjectUrl, string projectUrl, bool hasTopics, string[] topics) : base(user, respositoryLink, respositoryName, description, totalStars, totalForks, hasProjectUrl, projectUrl, hasTopics, topics) + { + Programminglanguage = programminglanguage; + HasDetails = hasDetails; + IsArchived = isArchived; + LastCommitTime = lastCommitTime; + LastCommitUrl = lastCommitUrl; + base.User = user; + base.RespositoryLink = respositoryLink; + base.RespositoryName = respositoryName; + base.Description = description; + base.TotalStars = totalStars; + base.TotalForks = totalForks; + base.HasProjectUrl = hasProjectUrl; + base.ProjectUrl = projectUrl; + base.HasTopics = hasTopics; + base.Topics = topics; + } + + public string Programminglanguage { get; set; } + + public bool HasDetails { get; set; } + + public bool IsArchived { get; set; } + + public string LastCommitTime { get; set; } + + public string LastCommitUrl { get; set; } + + } +} \ No newline at end of file diff --git a/GithubNetDemo/Program.cs b/GithubNetDemo/Program.cs index 9603642..d120681 100644 --- a/GithubNetDemo/Program.cs +++ b/GithubNetDemo/Program.cs @@ -1,4 +1,5 @@ using GithubNet; +using GithubNet.Models; namespace GithubNetDemo { @@ -8,8 +9,10 @@ private static async Task Main(string[] args) { GithubNetClient client = new(); - List testEntries = await client.GetTrendItemsAsync(); + Repository tinygrad = await client.GetRepositoryInfoAsync("https://github.com/liebki/GithubNet"); + Console.WriteLine(tinygrad.ToString()); + List testEntries = await client.GetTrendItemsAsync(); foreach (TrendItem entry in testEntries) { Console.WriteLine(); diff --git a/README.md b/README.md index 1a7ff18..ff29792 100644 --- a/README.md +++ b/README.md @@ -4,6 +4,7 @@ GithubNet is a C# library that allows you to retrieve trending GitHub repositories and their information. Currently, it provides functionality to fetch the trending repositories and their details. The library is built using .NET Core 7 and will be available as a NuGet package. + ## Technologies - C# @@ -11,40 +12,41 @@ GithubNet is a C# library that allows you to retrieve trending GitHub repositori - [HtmlAgilityPack](https://www.nuget.org/packages/HtmlAgilityPack) - [CssSelectors.Core.HtmlAgilityPack](https://www.nuget.org/packages/CssSelectors.Core.HtmlAgilityPack) + ## Features -- Retrieve trending GitHub repositories -- Access detailed information for each repository, including: - - User - - Repository link - - Repository name - - Description - - Total stars - - Total forks - - Programming language - - Availability of details - - Archival status - - Availability of project URL - - Project URL - - Availability of topics - - Topics - - Last commit time - - Last commit URL - -## Logo - -The logo, which is used for the nuget-package was created by: https://uxwing.com +Both the `Repository` and `TrendItem` classes inherit from a base class named `ItemBase`. The `ItemBase` class includes the following common properties: -## Usage +- `User` (string): The username or owner of the repository. +- `RespositoryLink` (string): The URL or link to the repository. +- `RespositoryName` (string): The name of the repository. +- `Description` (string): A brief description or summary of the repository. +- `TotalStars` (integer): Indicates the total number of stars or favorites received by the repository. +- `TotalForks` (integer): Represents the total number of forks or copies of the repository. +- `HasProjectUrl` (boolean): Indicates whether the repository has a project URL. +- `ProjectUrl` (string): The URL or link to the project associated with the repository, if available. +- `HasTopics` (boolean): Indicates whether the repository has topics associated with it. +- `Topics` (string[]): An array of strings representing the topics or tags associated with the repository. + +The `Repository` class includes additional properties specific to repositories: -To use the GithubNet library, follow these steps: +- `OpenIssuesNumber` (integer): Represents the number of open issues in the repository. +- `OpenPullRequestsNumber` (integer): Indicates the number of open pull requests in the repository. +- `TotalCommitsNumber` (integer): Represents the total number of commits made in the repository. +- `TotalContributorsNumber` (integer): Indicates the total number of contributors to the repository. -1. Create an instance of the `GithubNetClient` class. -2. Use the `GetTrendItemsAsync` method to retrieve a list of `TrendItem` objects representing trending repositories. You can optionally specify the `loadTrendItemDetails` parameter as `true` to load all the available details, but note that this process can be time-consuming as it crawls through each page. -3. If needed, you can use the `GetTrendItemDetailsAsync` method to fetch additional details for a specific `TrendItem` object. -4. Use the `GetTopicUrlFromTopicName` method to obtain the GitHub URL for a given topic. +The `TrendItem` class includes additional properties specific to trend items: -The following methods are described in the text: +- `Programminglanguage` (string): The main programming language associated with the trend item. +- `HasDetails` (boolean): Indicates whether it's an intense crawl, where each repository is individually crawled. +- `IsArchived` (boolean): Indicates whether the repository associated with the trend item is archived. +- `LastCommitTime` (string): The timestamp or time of the last commit made to the repository. +- `LastCommitUrl` (string): The URL or link to the last commit made in the repository. + + +## Usage + +The following methods can be used: - `GetTrendItemsAsync`: This method retrieves a list of `TrendItem` objects representing trending repositories. It returns the list asynchronously and can optionally load all available details if the `loadTrendItemDetails` parameter is set to `true`. @@ -52,27 +54,33 @@ The following methods are described in the text: - `GetTopicUrlFromTopicName`: This method allows you to obtain the GitHub URL for a given topic. It takes the topic name as input and returns the corresponding URL. +- `GetRepositoryInfoAsync`: This method retrieves detailed information about a specific GitHub repository. It provides details such as the number of contributors, the number of open issues, and more. The method returns the repository information asynchronously. + + ## Example For a demonstration of the library's functionality, refer to the included `GithubNetDemo` project. It showcases the console output of the crawled data using the `GetTrendItemsAsync` method. + ## License GithubNet is licensed under the GNU General Public License v3.0. You can read the full license details of the GNU General Public License v3.0 [here](https://choosealicense.com/licenses/gpl-3.0/). + ## Roadmap The roadmap for future development includes the following planned features: -1. `GetRepositoryInfoAsync`: Implement a method to retrieve detailed information about a specific GitHub repository. This method will provide details such as the repository owner, description, stars, forks, programming language, and other relevant information. +- Reduce the redundant code in the `GithubNetManager` class! + +- `GetUserInfoAsync`: Develop a method to fetch information about a GitHub user. This method will allow users to retrieve details such as the user's name, bio, location, profile picture, number of followers, number of repositories, and other relevant information. -2. `GetUserInfoAsync`: Develop a method to fetch information about a GitHub user. This method will allow users to retrieve details such as the user's name, bio, location, profile picture, number of followers, number of repositories, and other relevant information. +- `GetAllTopicsAsync`: Create a method to fetch all available topics from the GitHub Topics page (https://github.com/topics). This method will provide a list of topics along with their corresponding URLs and other relevant information. -3. `GetAllTopicsAsync`: Create a method to fetch all available topics from the GitHub Topics page (https://github.com/topics). This method will provide a list of topics along with their corresponding URLs and other relevant information. +- `GetAllCollectionsAsync`: Implement a method to retrieve information about the GitHub Collections page (https://github.com/collections). This method will allow users to obtain a list of collections, including their titles, descriptions, URLs, and other relevant details. -4. `GetAllCollectionsAsync`: Implement a method to retrieve information about the GitHub Collections page (https://github.com/collections). This method will allow users to obtain a list of collections, including their titles, descriptions, URLs, and other relevant details. ## Disclaimer