Skip to content

Commit b9aab3b

Browse files
author
Oleg Koloskov
committed
* Move to .net 8.0
* fix https://znanium.ru/
1 parent e8ca1bb commit b9aab3b

File tree

31 files changed

+59
-50
lines changed

31 files changed

+59
-50
lines changed

‎Common/Core/Core.csproj

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,14 @@
11
<Project Sdk="Microsoft.NET.Sdk">
22

33
<PropertyGroup>
4-
<TargetFramework>net7.0</TargetFramework>
4+
<TargetFramework>net8.0</TargetFramework>
55
</PropertyGroup>
66

77
<ItemGroup>
88
<PackageReference Include="CommandLineParser" Version="2.9.1" />
9-
<PackageReference Include="MongoDB.Driver" Version="2.19.0" />
9+
<PackageReference Include="MongoDB.Driver" Version="2.23.1" />
1010
<PackageReference Include="Ninject" Version="3.3.6" />
11-
<PackageReference Include="NLog" Version="5.1.2" />
11+
<PackageReference Include="NLog" Version="5.2.8" />
1212
</ItemGroup>
1313

1414
</Project>

‎Common/Parser.Core/Extensions/HttpClientExtensions.cs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ public static class HttpClientExtensions {
2222
}
2323

2424
if (response.StatusCode != HttpStatusCode.OK) {
25+
await Task.Delay(i * 1000);
2526
continue;
2627
}
2728

‎Common/Parser.Core/Parser.Core.csproj

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
<Project Sdk="Microsoft.NET.Sdk">
22

33
<PropertyGroup>
4-
<TargetFramework>net7.0</TargetFramework>
4+
<TargetFramework>net8.0</TargetFramework>
55
</PropertyGroup>
66

77
<ItemGroup>
@@ -15,7 +15,7 @@
1515
</ItemGroup>
1616

1717
<ItemGroup>
18-
<PackageReference Include="HtmlAgilityPack" Version="1.11.46" />
18+
<PackageReference Include="HtmlAgilityPack" Version="1.11.59" />
1919
<PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
2020
</ItemGroup>
2121

‎Comparer/Book.Comparer.Logic/Book.Comparer.Logic.csproj

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
<Project Sdk="Microsoft.NET.Sdk">
22

33
<PropertyGroup>
4-
<TargetFramework>net7.0</TargetFramework>
4+
<TargetFramework>net8.0</TargetFramework>
55
</PropertyGroup>
66

77
<ItemGroup>

‎Comparer/Book.Comparer.Logic/Logic/BibParser.cs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -13,10 +13,10 @@ public class BibParser {
1313
private readonly Normalizer _normalizer;
1414
private readonly BibParserConfig _config;
1515

16-
private static readonly Regex[] AuthorRegexs = {
16+
private static readonly Regex[] AuthorRegexs = [
1717
new(@"(\b|\s)(?<SecondName>\w+)(?<io>\s\w\.){1,2}(\b|\s)", RegexOptions.Compiled),
1818
new(@"(\b|\s)(?<io>\w\.\s){1,2}(?<SecondName>\w+)(\b|\s)", RegexOptions.Compiled)
19-
};
19+
];
2020

2121
public BibParser(Normalizer normalizer, BibParserConfig config) {
2222
_normalizer = normalizer;

‎Comparer/Book.Comparer.Logic/Utils/Normalizer.cs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ public class Normalizer {
1212
private static readonly Regex _nonCharacter = new("\\W", RegexOptions.Compiled | RegexOptions.IgnoreCase);
1313
private static readonly Regex _nonDigits = new("\\D", RegexOptions.Compiled | RegexOptions.IgnoreCase);
1414

15-
public readonly string[] AuthorsSeparator = { ",", ";", ":" };
15+
public readonly string[] AuthorsSeparator = [",", ";", ":"];
1616
public readonly HashSet<string> NonSingAuthorWords;
1717
public readonly HashSet<string> NonSignBibWords;
1818

@@ -82,7 +82,7 @@ public string RemoveNonCharacters(string str) {
8282
/// <param name="str"></param>
8383
/// <returns></returns>
8484
public IEnumerable<string> SplitWords(string str) {
85-
return string.IsNullOrWhiteSpace(str) ? new string[] { } : _nonCharacter.Split(str);
85+
return string.IsNullOrWhiteSpace(str) ? [] : _nonCharacter.Split(str);
8686
}
8787

8888
/// <summary>

‎Comparer/Book.Comparer/Book.Comparer.csproj

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22

33
<PropertyGroup>
44
<OutputType>Exe</OutputType>
5-
<TargetFramework>net7.0</TargetFramework>
5+
<TargetFramework>net8.0</TargetFramework>
66
</PropertyGroup>
77

88
<ItemGroup>
@@ -19,9 +19,9 @@
1919
</ItemGroup>
2020

2121
<ItemGroup>
22-
<PackageReference Include="Microsoft.Extensions.Configuration" Version="7.0.0" />
23-
<PackageReference Include="Microsoft.Extensions.Configuration.Binder" Version="7.0.4" />
24-
<PackageReference Include="Microsoft.Extensions.Configuration.Json" Version="7.0.0" />
22+
<PackageReference Include="Microsoft.Extensions.Configuration" Version="8.0.0" />
23+
<PackageReference Include="Microsoft.Extensions.Configuration.Binder" Version="8.0.1" />
24+
<PackageReference Include="Microsoft.Extensions.Configuration.Json" Version="8.0.0" />
2525
</ItemGroup>
2626

2727
</Project>

‎Parsers/AcademiaMoscow.Parser/AcademiaMoscow.Parser.csproj

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22

33
<PropertyGroup>
44
<OutputType>Exe</OutputType>
5-
<TargetFramework>net7.0</TargetFramework>
5+
<TargetFramework>net8.0</TargetFramework>
66
</PropertyGroup>
77

88
<ItemGroup>

‎Parsers/AcademiaMoscow.Parser/Logic/Parser.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,7 @@ protected override async Task<IDataflowBlock[]> RunInternal(HttpClient client, I
4343
await getPageBlock.SendAsync(GetUrl(i));
4444
}
4545

46-
return new IDataflowBlock[] {getPageBlock, filterBlock, getBookBlock, batchBlock, saveBookBlock};
46+
return [getPageBlock, filterBlock, getBookBlock, batchBlock, saveBookBlock];
4747
}
4848

4949
private async Task<BookInfo> GetBook(HttpClient client, Uri uri) {

‎Parsers/BiblioClub.Parser/BiblioClub.Parser.csproj

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22

33
<PropertyGroup>
44
<OutputType>Exe</OutputType>
5-
<TargetFramework>net7.0</TargetFramework>
5+
<TargetFramework>net8.0</TargetFramework>
66
</PropertyGroup>
77

88
<ItemGroup>

‎Parsers/BiblioClub.Parser/Logic/Parser.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,7 @@ protected override async Task<IDataflowBlock[]> RunInternal(HttpClient client, I
4848
await batchBlock1.SendAsync(i);
4949
}
5050

51-
return new IDataflowBlock[] {batchBlock1, getPageBlock, filterBlock, batchBlock2, getBibBlock, batchBlock3, saveBookBlock};
51+
return [batchBlock1, getPageBlock, filterBlock, batchBlock2, getBibBlock, batchBlock3, saveBookBlock];
5252
}
5353

5454
private static IEnumerable<BookShortInfo> Filter(IEnumerable<BookShortInfo> shortInfos, ISet<string> processed) {

‎Parsers/BiblioRossica.Parser/BiblioRossica.Parser.csproj

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22

33
<PropertyGroup>
44
<OutputType>Exe</OutputType>
5-
<TargetFramework>net7.0</TargetFramework>
5+
<TargetFramework>net8.0</TargetFramework>
66
</PropertyGroup>
77

88
<ItemGroup>

‎Parsers/BiblioRossica.Parser/Logic/Parser.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,7 @@ protected override async Task<IDataflowBlock[]> RunInternal(HttpClient client, I
4949
await getPageBlock.SendAsync(i);
5050
}
5151

52-
return new IDataflowBlock[]{getPageBlock, filterBlock, getBookBlock, batchBlock, saveBookBlock};
52+
return [getPageBlock, filterBlock, getBookBlock, batchBlock, saveBookBlock];
5353
}
5454

5555
private static IEnumerable<string> Filter(BookItems bookItems, ISet<string> processed) {

‎Parsers/BookRu.Parser/BookRu.Parser.csproj

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22

33
<PropertyGroup>
44
<OutputType>Exe</OutputType>
5-
<TargetFramework>net7.0</TargetFramework>
5+
<TargetFramework>net8.0</TargetFramework>
66
</PropertyGroup>
77

88
<ItemGroup>

‎Parsers/BookRu.Parser/Logic/Parser.cs

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
using System;
22
using System.Collections.Generic;
33
using System.Linq;
4+
using System.Net;
45
using System.Net.Http;
56
using System.Text.Json;
67
using System.Threading.Tasks;
@@ -46,7 +47,7 @@ protected override async Task<IDataflowBlock[]> RunInternal(HttpClient client, I
4647
await getBookIdsBlock.SendAsync(categoryId);
4748
}
4849

49-
return new IDataflowBlock[] {getBookIdsBlock, filterBlock, getBooksBlock, batchBlock, saveBookBlock};
50+
return [getBookIdsBlock, filterBlock, getBooksBlock, batchBlock, saveBookBlock];
5051
}
5152

5253
private async Task<JsonDocument> GetNextData(HttpClient client) {
@@ -86,9 +87,12 @@ private static async Task<IEnumerable<string>> GetBookIds(HttpClient client, str
8687
}
8788

8889
private async Task<BookInfo> GetBook(HttpClient client, string buildId, string id) {
89-
var response = await client.GetStringAsync(new Uri($"https://book.ru/_next/data/{buildId}/book/{id}.json?"));
90-
var json = JsonDocument.Parse(response);
91-
90+
var response = await client.GetAsync(new Uri($"https://book.ru/_next/data/{buildId}/book/{id}.json?"));
91+
if (response.StatusCode != HttpStatusCode.OK) {
92+
return default;
93+
}
94+
95+
var json = JsonDocument.Parse(await response.Content.ReadAsStringAsync());
9296
var items = JsonConvert.DeserializeObject<BookItem[]>(json.RootElement.GetProperty("pageProps").GetProperty("serverDataBook").GetProperty("item").GetRawText());
9397

9498
if (items.Length == 0) {

‎Parsers/IBooks.Parser/IBooks.Parser.csproj

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22

33
<PropertyGroup>
44
<OutputType>Exe</OutputType>
5-
<TargetFramework>net7.0</TargetFramework>
5+
<TargetFramework>net8.0</TargetFramework>
66
</PropertyGroup>
77

88
<ItemGroup>

‎Parsers/IBooks.Parser/Logic/Parser.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@ protected override async Task<IDataflowBlock[]> RunInternal(HttpClient client, I
4141
await getBooksBlock.SendAsync(i);
4242
}
4343

44-
return new IDataflowBlock[] {getBooksBlock, filterBlock, batchBlock, saveBookBlock};
44+
return [getBooksBlock, filterBlock, batchBlock, saveBookBlock];
4545
}
4646

4747
private static IEnumerable<BookInfo> Filter(IEnumerable<BookInfo> books, ISet<string> processed) {

‎Parsers/IprBookShop.Parser/IprBookShop.Parser.csproj

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22

33
<PropertyGroup>
44
<OutputType>Exe</OutputType>
5-
<TargetFramework>net7.0</TargetFramework>
5+
<TargetFramework>net8.0</TargetFramework>
66
</PropertyGroup>
77

88
<ItemGroup>

‎Parsers/IprBookShop.Parser/Logic/Parser.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -52,7 +52,7 @@ protected override async Task<IDataflowBlock[]> RunInternal(HttpClient client, I
5252
await getPageBlock.SendAsync(i);
5353
}
5454

55-
return new IDataflowBlock[] {getPageBlock, filterBlock, getBookBlock, batchBlock, saveBookBlock};
55+
return [getPageBlock, filterBlock, getBookBlock, batchBlock, saveBookBlock];
5656
}
5757

5858
private static string Normalize(string str) {

‎Parsers/LanBook.Parser/LanBook.Parser.csproj

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22

33
<PropertyGroup>
44
<OutputType>Exe</OutputType>
5-
<TargetFramework>net7.0</TargetFramework>
5+
<TargetFramework>net8.0</TargetFramework>
66
</PropertyGroup>
77

88
<ItemGroup>

‎Parsers/LanBook.Parser/Logic/Parser.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -65,7 +65,7 @@ protected override async Task<IDataflowBlock[]> RunInternal(HttpClient client, I
6565
await getPageBlock.SendAsync(i);
6666
}
6767

68-
return new IDataflowBlock[] {getPageBlock, filterBlock, getBookBlock, batchBlock, saveBookBlock};
68+
return [getPageBlock, filterBlock, getBookBlock, batchBlock, saveBookBlock];
6969
}
7070

7171
private static IEnumerable<long> Filter(ApiResponse<BooksShortBody> response, ISet<string> processed) {

‎Parsers/ProfSpo.Parser/Logic/Parser.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,7 @@ protected override async Task<IDataflowBlock[]> RunInternal(HttpClient client, I
4646
await getBooksBlock.SendAsync(i);
4747
}
4848

49-
return new IDataflowBlock[] {getBooksBlock, filterBlock, batchBlock, saveBookBlock};
49+
return [getBooksBlock, filterBlock, batchBlock, saveBookBlock];
5050
}
5151

5252
private static IEnumerable<BookInfo> Filter(IEnumerable<BookInfo> books, ISet<string> processed) {

‎Parsers/ProfSpo.Parser/ProfSpo.Parser.csproj

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22

33
<PropertyGroup>
44
<OutputType>Exe</OutputType>
5-
<TargetFramework>net7.0</TargetFramework>
5+
<TargetFramework>net8.0</TargetFramework>
66
</PropertyGroup>
77

88
<ItemGroup>

‎Parsers/RuCont.Parser/Logic/Parser.cs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,7 @@ protected override async Task<IDataflowBlock[]> RunInternal(HttpClient client, I
4343
await getPageBlock.SendAsync(sitemap.Location);
4444
}
4545

46-
return new IDataflowBlock[] {getPageBlock, filterBlock, getBookBlock, batchBlock, saveBookBlock};
46+
return [getPageBlock, filterBlock, getBookBlock, batchBlock, saveBookBlock];
4747
}
4848

4949
private static IEnumerable<string> Filter(SitemapFile sitemap, ISet<string> processed) {
@@ -71,7 +71,7 @@ private async Task<BookInfo> GetBook(HttpClient client, string id) {
7171
}
7272

7373
var authors = new List<string>();
74-
foreach (var author in book.Authors ?? new string[]{ }) {
74+
foreach (var author in book.Authors ?? []) {
7575
var cleanAuthor = Regex.Replace(author, @"\((.*?)\)", string.Empty).Trim();
7676
if (!string.IsNullOrWhiteSpace(book.Bib)) {
7777
try {

‎Parsers/RuCont.Parser/RuCont.Parser.csproj

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22

33
<PropertyGroup>
44
<OutputType>Exe</OutputType>
5-
<TargetFramework>net7.0</TargetFramework>
5+
<TargetFramework>net8.0</TargetFramework>
66
</PropertyGroup>
77

88
<ItemGroup>

‎Parsers/StudentLibrary.Parser/Logic/Parser.cs

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -54,8 +54,8 @@ private async Task<BookInfo> GetBook(HttpClient client, Uri uri) {
5454
.Split(' ')
5555
.First(), out book.Pages);
5656
if(value.Contains("ISBN")) {
57-
book.ISBN = value.Split(new []{ "ISBN"}, StringSplitOptions.None)[1].Trim()
58-
.Split(new []{ ", " }, StringSplitOptions.None)
57+
book.ISBN = value.Split(["ISBN"], StringSplitOptions.None)[1].Trim()
58+
.Split([", "], StringSplitOptions.None)
5959
.First();
6060
}
6161
} else if (name.Contains("Издательство")) {
@@ -125,7 +125,7 @@ protected override async Task<IDataflowBlock[]> RunInternal(HttpClient client, I
125125
await getPageBlock.SendAsync(GetUrl(i));
126126
}
127127

128-
return new IDataflowBlock[] {getPageBlock, filterBlock, getBookBlock, batchBlock, saveBookBlock };
128+
return [getPageBlock, filterBlock, getBookBlock, batchBlock, saveBookBlock];
129129
}
130130
}
131131
}

‎Parsers/StudentLibrary.Parser/StudentLibrary.Parser.csproj

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22

33
<PropertyGroup>
44
<OutputType>Exe</OutputType>
5-
<TargetFramework>net7.0</TargetFramework>
5+
<TargetFramework>net8.0</TargetFramework>
66
</PropertyGroup>
77

88
<ItemGroup>

‎Parsers/Urait.Parser/Logic/Parser.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,7 @@ protected override async Task<IDataflowBlock[]> RunInternal(HttpClient client, I
3939

4040
await filterBlock.SendAsync(await GetLinksSitemaps(client, new Uri("https://urait.ru/sitemap.xml")));
4141

42-
return new IDataflowBlock[] {filterBlock, getBookBlock, batchBlock, saveBookBlock};
42+
return [filterBlock, getBookBlock, batchBlock, saveBookBlock];
4343
}
4444

4545
private async Task<BookInfo> GetBook(HttpClient client, Uri uri) {

‎Parsers/Urait.Parser/Urait.Parser.csproj

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22

33
<PropertyGroup>
44
<OutputType>Exe</OutputType>
5-
<TargetFramework>net7.0</TargetFramework>
5+
<TargetFramework>net8.0</TargetFramework>
66
</PropertyGroup>
77

88
<ItemGroup>

‎Parsers/Znanium.Parser/Logic/Parser.cs

Lines changed: 11 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -45,23 +45,27 @@ protected override async Task<IDataflowBlock[]> RunInternal(HttpClient client, I
4545
await getPageBlock.SendAsync(sitemap.Location);
4646
}
4747

48-
return new IDataflowBlock[]{getPageBlock, filterBlock, getBookBlock, batchBlock, saveBookBlock};
48+
return [getPageBlock, filterBlock, getBookBlock, batchBlock, saveBookBlock];
4949
}
5050

5151
private async Task<BookInfo> GetBook(HttpClient client, long id) {
52-
var (response, statusCode) = await client.GetStringWithTriesAsync(new Uri($"https://znanium.com/catalog/document?id={id}"));
53-
54-
var book = new BookInfo(id.ToString(), ElsName);
52+
var url = new Uri($"https://znanium.com/catalog/document?id={id}");
53+
var (response, statusCode) = await client.GetStringWithTriesAsync(url);
54+
5555
if (statusCode == HttpStatusCode.NotFound || response == default) {
56-
return book;
56+
_logger.Warn($"Не удалось загрузить книгу {url}. {statusCode}");
57+
return default;
5758
}
58-
59+
60+
_logger.Debug($"Получена книга {url}");
61+
62+
var book = new BookInfo(id.ToString(), ElsName);
5963
var doc = new HtmlDocument();
6064
doc.LoadHtml(response);
6165

6266
var bookContent = doc.DocumentNode.GetByFilterFirst("div", "book-single__top-content-right");
6367
book.Name = doc.DocumentNode.GetByFilterFirst("h1", "book-single__title")?.InnerText.Trim();
64-
book.Bib = doc.DocumentNode.GetByFilter("div", "tab-panel").FirstOrDefault(t => t.GetAttributeValue("nav", string.Empty) == "bib")?.InnerText;
68+
book.Bib = doc.DocumentNode.GetByFilter("div", "tab-panel").FirstOrDefault(t => t.GetAttributeValue("nav", string.Empty) == "bib")?.InnerText?.Trim();
6569

6670
foreach (var div in bookContent.GetByFilter("div", "book-link")) {
6771
var name = div.InnerText.Trim();

‎Parsers/Znanium.Parser/Znanium.Parser.csproj

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22

33
<PropertyGroup>
44
<OutputType>Exe</OutputType>
5-
<TargetFramework>net7.0</TargetFramework>
5+
<TargetFramework>net8.0</TargetFramework>
66
</PropertyGroup>
77

88
<ItemGroup>

0 commit comments

Comments
 (0)