Skip to content

Commit

Permalink
Fixes #85903 #85910 Google search does not work (API and web)
Browse files Browse the repository at this point in the history
  • Loading branch information
@iosebyte
Ioseba Palop authored and iosebyte committed Jun 18, 2019
1 parent e881a43 commit 0175c30
Show file tree
Hide file tree
Showing 14 changed files with 166 additions and 283 deletions.
19 changes: 18 additions & 1 deletion FOCA/FOCA.csproj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,9 @@
<WarningLevel>4</WarningLevel>
<Prefer32Bit>false</Prefer32Bit>
</PropertyGroup>
<PropertyGroup>
<AutoGenerateBindingRedirects>true</AutoGenerateBindingRedirects>
</PropertyGroup>
<ItemGroup>
<Reference Include="com.rusanu.dataconnectiondialog, Version=1.0.0.1, Culture=neutral, processorArchitecture=MSIL">
<HintPath>..\packages\com.rusanu.dataconnectiondialog.1.0.0.1\lib\net20\com.rusanu.dataconnectiondialog.dll</HintPath>
Expand Down Expand Up @@ -118,11 +121,25 @@
<Reference Include="System.Core">
<RequiredTargetFramework>3.5</RequiredTargetFramework>
</Reference>
<Reference Include="System.Net.Http" />
<Reference Include="System.Net.Http, Version=4.1.1.3, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
<HintPath>..\packages\System.Net.Http.4.3.4\lib\net46\System.Net.Http.dll</HintPath>
</Reference>
<Reference Include="System.Runtime.Serialization">
<RequiredTargetFramework>3.0</RequiredTargetFramework>
</Reference>
<Reference Include="System.Security" />
<Reference Include="System.Security.Cryptography.Algorithms, Version=4.1.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
<HintPath>..\packages\System.Security.Cryptography.Algorithms.4.3.0\lib\net461\System.Security.Cryptography.Algorithms.dll</HintPath>
</Reference>
<Reference Include="System.Security.Cryptography.Encoding, Version=4.0.1.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
<HintPath>..\packages\System.Security.Cryptography.Encoding.4.3.0\lib\net46\System.Security.Cryptography.Encoding.dll</HintPath>
</Reference>
<Reference Include="System.Security.Cryptography.Primitives, Version=4.0.1.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
<HintPath>..\packages\System.Security.Cryptography.Primitives.4.3.0\lib\net46\System.Security.Cryptography.Primitives.dll</HintPath>
</Reference>
<Reference Include="System.Security.Cryptography.X509Certificates, Version=4.1.1.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
<HintPath>..\packages\System.Security.Cryptography.X509Certificates.4.3.0\lib\net461\System.Security.Cryptography.X509Certificates.dll</HintPath>
</Reference>
<Reference Include="System.ServiceModel">
<RequiredTargetFramework>3.0</RequiredTargetFramework>
</Reference>
Expand Down
10 changes: 3 additions & 7 deletions FOCA/PanelDNSSearch.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@
using Heijden.DNS;
using MetadataExtractCore;
using MetadataExtractCore.Diagrams;
using SearcherCore.Searcher.BingAPI;
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
Expand Down Expand Up @@ -472,8 +471,7 @@ private void GoogleApiSearch()

try
{
var gr = (GoogleAPISearcher.GoogleAPIResults)item;
var url = new Uri(gr.Url);
var url = new Uri((string)item);
var strHost = url.Host;
if (
currentResults.All(
Expand Down Expand Up @@ -687,8 +685,7 @@ private void BingApiSearch()

try
{
var br = (BingApiResult)item;
var url = new Uri(br.Url);
var url = new Uri((string)item);
var strHost = url.Host;
if (
currentResults.All(
Expand Down Expand Up @@ -905,8 +902,7 @@ private void SerchLinkApiBingEvent(string ip, BingAPISearcher bingSearcherApi, L
loopState.Stop();
try
{
var br = (BingApiResult)item;
var url = new Uri(br.Url);
var url = new Uri((string)item);
if (
currentResults.Any(d => string.Equals(d, url.Host, StringComparison.CurrentCultureIgnoreCase)))
return;
Expand Down
57 changes: 53 additions & 4 deletions FOCA/PanelMetadataSearch.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2385,10 +2385,35 @@ private void CustomSearch(object parameter)
var searchString = parameter as string;
try
{

if (chkGoogle.Checked)
CustomSearchEventsGeneric(new GoogleWebSearcher(), searchString);
{
if (String.IsNullOrWhiteSpace(Program.cfgCurrent.GoogleApiKey) || String.IsNullOrWhiteSpace(Program.cfgCurrent.GoogleApiCx))
{
CustomSearchEventsGeneric(new GoogleWebSearcher(), searchString);
}
else
{
CustomSearchEventsGeneric(new GoogleAPISearcher
{
GoogleApiKey = Program.cfgCurrent.GoogleApiKey,
GoogleApiCx = Program.cfgCurrent.GoogleApiCx,
SearchAll = true
}, searchString);
}
}

if (chkBing.Checked)
CustomSearchEventsGeneric(new BingWebSearcher(), searchString);
{
if (String.IsNullOrWhiteSpace(Program.cfgCurrent.BingApiKey))
{
CustomSearchEventsGeneric(new BingWebSearcher(), searchString);
}
else
{
CustomSearchEventsGeneric(new BingAPISearcher(Program.cfgCurrent.BingApiKey), searchString);
}
}
}
catch (ThreadAbortException)
{
Expand Down Expand Up @@ -2437,9 +2462,33 @@ private void SearchAll()
try
{
if (chkGoogle.Checked)
SearchEventsGeneric(new GoogleWebSearcher());
{
if (String.IsNullOrWhiteSpace(Program.cfgCurrent.GoogleApiKey) || String.IsNullOrWhiteSpace(Program.cfgCurrent.GoogleApiCx))
{
SearchEventsGeneric(new GoogleWebSearcher());
}
else
{
SearchEventsGeneric(new GoogleAPISearcher
{
GoogleApiKey = Program.cfgCurrent.GoogleApiKey,
GoogleApiCx = Program.cfgCurrent.GoogleApiCx,
SearchAll = true
});
}
}

if (chkBing.Checked)
SearchEventsGeneric(new BingWebSearcher());
{
if (String.IsNullOrWhiteSpace(Program.cfgCurrent.BingApiKey))
{
SearchEventsGeneric(new BingWebSearcher());
}
else
{
SearchEventsGeneric(new BingAPISearcher(Program.cfgCurrent.BingApiKey));
}
}
if (chkDuck.Checked)
SearchEventsGeneric(new DuckduckgoWebSearcher());
}
Expand Down
4 changes: 2 additions & 2 deletions FOCA/Properties/AssemblyInfo.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,5 +32,5 @@
// You can specify all the values or you can default the Build and Revision Numbers
// by using the '*' as shown below:
// [assembly: AssemblyVersion("1.0.*")]
[assembly: AssemblyVersion("3.4.4.1")]
[assembly: AssemblyFileVersion("3.4.4.1")]
[assembly: AssemblyVersion("3.4.5.0")]
[assembly: AssemblyFileVersion("3.4.5.0")]
4 changes: 2 additions & 2 deletions FOCA/app.config
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
<supportedRuntime version="v4.0" sku=".NETFramework,Version=v4.6.1" />
</startup>
<connectionStrings>
<add name="FocaContextDb" connectionString="Server=.\SQLEXPRESS;Initial Catalog=Foca;MultipleActiveResultSets=True;Integrated Security=true;Connection Timeout=3" providerName="System.Data.SQLCLient"/>
<add name="FocaContextDb" connectionString="Server=.\SQLEXPRESS;Initial Catalog=Foca;MultipleActiveResultSets=True;Integrated Security=true;Connection Timeout=3" providerName="System.Data.SQLCLient" />
</connectionStrings>
<entityFramework>
<defaultConnectionFactory type="System.Data.Entity.Infrastructure.SqlConnectionFactory, EntityFramework" />
Expand All @@ -20,7 +20,7 @@
<assemblyBinding xmlns="urn:schemas-microsoft-com:asm.v1">
<dependentAssembly>
<assemblyIdentity name="System.Net.Http" publicKeyToken="b03f5f7f11d50a3a" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-4.1.1.0" newVersion="4.1.1.0" />
<bindingRedirect oldVersion="0.0.0.0-4.1.1.3" newVersion="4.1.1.3" />
</dependentAssembly>
</assemblyBinding>
</runtime>
Expand Down
5 changes: 5 additions & 0 deletions FOCA/packages.config
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,4 +7,9 @@
<package id="NLog" version="4.4.9" targetFramework="net461" />
<package id="NLog.Config" version="4.4.9" targetFramework="net461" />
<package id="NLog.Schema" version="4.4.9" targetFramework="net461" />
<package id="System.Net.Http" version="4.3.4" targetFramework="net461" />
<package id="System.Security.Cryptography.Algorithms" version="4.3.0" targetFramework="net461" />
<package id="System.Security.Cryptography.Encoding" version="4.3.0" targetFramework="net461" />
<package id="System.Security.Cryptography.Primitives" version="4.3.0" targetFramework="net461" />
<package id="System.Security.Cryptography.X509Certificates" version="4.3.0" targetFramework="net461" />
</packages>
43 changes: 11 additions & 32 deletions SearcherCore/SearcherCore/Searcher/BingAPI/SearchBingApi.cs
View file
Original file line number Diff line number Diff line change
@@ -1,29 +1,11 @@
using System;
using Newtonsoft.Json.Linq;
using RestSharp;
using System;
using System.Collections.Generic;
using System.Linq;
using Newtonsoft.Json.Linq;
using RestSharp;

namespace SearcherCore.Searcher.BingAPI
{
/// <summary>
/// BingApiResults encapsulates the fields of each result
/// that FOCA may use
/// </summary>
public class BingApiResult
{
public BingApiResult(string url, string title, string description)
{
Url = url;
Title = title;
Description = description;
}

public string Url { get; set; }
public string Title { get; set; }
public string Description { get; set; }
}

public class SearchBingApi
{
public delegate void StatusUpdateHandler(object sender, string e);
Expand All @@ -46,9 +28,9 @@ public SearchBingApi(string secretKey)
/// </summary>
/// <param name="q">query</param>
/// <returns></returns>
public List<BingApiResult> Search(string q)
public List<string> Search(string q)
{
var results = new List<BingApiResult>();
var results = new List<string>();
var offset = 0;
// URL of the requests. Count = 50 because it's the max allowed value
var request = new RestRequest($"search?count=50&safeSearch=Off&textFormat=Raw&offset={offset}&q={q}",
Expand All @@ -62,21 +44,18 @@ public List<BingApiResult> Search(string q)
{
do
{
var webpages = (JArray) token["webPages"]["value"];
var webpages = (JArray)token["webPages"]["value"];
results.AddRange(
webpages.Select(
res =>
new BingApiResult((string) res.SelectToken("displayUrl"),
(string) res.SelectToken("name"), (string) res.SelectToken("snippet"))));
foreach (
var b in
res => (string)res.SelectToken("displayUrl")));
foreach (var b in
webpages.Select(
link => new BingApiResult((string) link.SelectToken("displayUrl"), "", "")))
link => ((string)link.SelectToken("displayUrl"))))
{
UpdateStatus(b.Url);
UpdateStatus(b);
}
offset += 50;
} while (offset < (int) token.SelectToken("webPages").SelectToken("totalEstimatedMatches")/1000);
} while (offset < (int)token.SelectToken("webPages").SelectToken("totalEstimatedMatches") / 1000);
}
catch
{
Expand Down
2 changes: 1 addition & 1 deletion SearcherCore/SearcherCore/Searcher/BingAPISearcher.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -110,7 +110,7 @@ private void GetCustomLinksAsync(object customSearchString)
private int GetBingResults(string searchString, out bool moreResults)
{
var client = new SearchBingApi(BingApiKey);
List<BingApiResult> results;
List<string> results;
try
{
results = client.Search(searchString);
Expand Down
6 changes: 3 additions & 3 deletions SearcherCore/SearcherCore/Searcher/GoogleAPI/SearchGoogleApi.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,11 +44,11 @@ private CseResource.ListRequest BuildRequest(string searchString)
return listRequest;
}

public List<GoogleAPISearcher.GoogleAPIResults> RunService(string searchString)
public List<string> RunService(string searchString)
{
var listRequest = BuildRequest(searchString);
IList<Result> paging = new List<Result>();
var urls = new List<GoogleAPISearcher.GoogleAPIResults>();
var urls = new List<string>();
var count = 0;
while (paging != null)
{
Expand All @@ -60,7 +60,7 @@ private CseResource.ListRequest BuildRequest(string searchString)
{
urls.AddRange(
paging.Select(
item => new GoogleAPISearcher.GoogleAPIResults {Url = item.Link, Title = item.Title}));
item => item.Link));
foreach (var item in paging)
{
UpdateStatus(item.Link);
Expand Down
24 changes: 7 additions & 17 deletions SearcherCore/SearcherCore/Searcher/GoogleAPISearcher.cs
View file
Original file line number Diff line number Diff line change
@@ -1,22 +1,12 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading;
using FOCA.Threads;
using SearcherCore.Searcher.GoogleAPI;
using System.Linq;
using System.Threading;

namespace FOCA.Searcher
{
public class GoogleAPISearcher: WebSearcher
public class GoogleAPISearcher : WebSearcher
{
public class GoogleAPIResults
{
public string Url;
public string Title;
public string UrlCache;
}

public string GoogleApiKey { get; set; }
public string GoogleApiCx { get; set; }
public const int maxResultPerRequest = 8;
Expand All @@ -38,7 +28,7 @@ public override void GetLinks()
if (thrSearchLinks != null && thrSearchLinks.IsAlive) return;
thrSearchLinks = new Thread(GetLinksAsync)
{
Priority = ThreadPriority.Lowest,
Priority = ThreadPriority.Lowest,
IsBackground = true
};
thrSearchLinks.Start();
Expand Down Expand Up @@ -68,7 +58,7 @@ private void GetLinksAsync()
OnSearcherChangeStateEvent(new EventsThreads.ThreadStringEventArgs("Searching links in " + Name + "..."));
try
{
foreach(var strExtension in Extensions)
foreach (var strExtension in Extensions)
{
OnSearcherChangeStateEvent(new EventsThreads.ThreadStringEventArgs("Search " + strExtension + " in " + Name));
GetGoogleLinks("site:" + Site + " filetype:" + strExtension);
Expand Down Expand Up @@ -96,7 +86,7 @@ private void GetCustomLinksAsync(object customSearchString)
try
{
if (SearchAll)
OnSearcherEndEvent(GetGoogleAllLinks((string) customSearchString) > maxResults - 10
OnSearcherEndEvent(GetGoogleAllLinks((string)customSearchString) > maxResults - 10
? new EventsThreads.ThreadEndEventArgs(
EventsThreads.ThreadEndEventArgs.EndReasonEnum.LimitReached)
: new EventsThreads.ThreadEndEventArgs(EventsThreads.ThreadEndEventArgs.EndReasonEnum.NoMoreData));
Expand Down Expand Up @@ -140,7 +130,7 @@ private int GetGoogleResults(string searchString, out bool moreResults)
OnSearcherLogEvent(new EventsThreads.ThreadStringEventArgs($"[{strName}] Found {results.Count} links"));
OnSearcherLinkFoundEvent(new EventsThreads.ThreadListDataFoundEventArgs(results));

return results.Count ;
return results.Count;
}

/// <summary>
Expand Down
Loading

0 comments on commit 0175c30

Please sign in to comment.