From 5a9072ec35fe740607022c4eb561d372bee5ab2a Mon Sep 17 00:00:00 2001 From: Bert Scholten Date: Thu, 30 Nov 2023 17:26:39 +0100 Subject: [PATCH 1/2] Attempt at improving suggestions from Bing map search The locations endpoint has a query part, but that doesn't really work as suggestions: it only returns results that are pretty much exactly what you're searching for, and something like 'edin' won't result anything. There is a autosuggest endpoint, but that one doesn't return geo information... So tying the 2 together, first the suggestion for sensible suggstions and then the locations endpoint to get geo information. Does mean more roundtrips, but there's a max of 10 results anyway. --- .../search/tasks/BingSearchService.java | 140 ++++++++++++++---- .../search/tasks/BingSearchServiceTest.java | 4 +- 2 files changed, 116 insertions(+), 28 deletions(-) diff --git a/search-service-bing-geocoder/src/main/java/nl/aerius/search/tasks/BingSearchService.java b/search-service-bing-geocoder/src/main/java/nl/aerius/search/tasks/BingSearchService.java index cd293cc..a4e5257 100644 --- a/search-service-bing-geocoder/src/main/java/nl/aerius/search/tasks/BingSearchService.java +++ b/search-service-bing-geocoder/src/main/java/nl/aerius/search/tasks/BingSearchService.java @@ -18,7 +18,11 @@ import java.util.ArrayList; import java.util.HashMap; +import java.util.LinkedHashSet; import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.stream.Collectors; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -47,8 +51,22 @@ public class BingSearchService implements SearchTaskService { private static final String REGION = "GB"; private static final String CULTURE = "en-GB"; private static final String BNG_BOUNDS = "49.79,-8.82,60.94,1.92"; - private static final String PDOK_SUGGEST_ENDPOINT = "https://dev.virtualearth.net/REST/v1/Locations?query=%s" - + "&key=%s&include=ciso2&userRegion=" + REGION + "&c=" + CULTURE + "&userMapView=" + BNG_BOUNDS; + private static final String BING_SUGGEST_ENDPOINT = "https://dev.virtualearth.net/REST/v1/Autosuggest?query=%s" + + "&key=%s" + + "&userRegion=" + REGION + + "&countryFilter=" + REGION + + "&includeEntityTypes=Address,Place" + + "&c=" + CULTURE + + "&userMapView=" + BNG_BOUNDS + + "&maxResults=10"; + private static final String BING_lOCATIONS_ENDPOINT = "https://dev.virtualearth.net/REST/v1/Locations?" + + "key=%s" + + "&userRegion=" + REGION + + "&countryRegion=" + REGION + + "&c=" + CULTURE + + "&userMapView=" + BNG_BOUNDS + + "&maxResults=1" + + "%s"; private static final Logger LOG = LoggerFactory.getLogger(BingSearchService.class); @@ -76,29 +94,58 @@ public Single retrieveSearchResults(final String query) { } private void retrieveSuggestions(final String query, final Map sugs) { - final String url = String.format(PDOK_SUGGEST_ENDPOINT, query, apiKey); + final String url = String.format(BING_SUGGEST_ENDPOINT, query, apiKey); + // First obtain suggestions, then translate them to actual locations. final HttpResponse json = Unirest.get(url).asJson(); final JSONObject body = json.getBody().getObject(); - final JSONArray arr = body.getJSONArray("resourceSets").getJSONObject(0).getJSONArray("resources"); + final JSONArray arr = body + .getJSONArray("resourceSets").getJSONObject(0) + .getJSONArray("resources").getJSONObject(0) + .getJSONArray("value"); + // As the list of suggestions can contain duplicates when we translate them to actual locations, filter those out by using a set + // Use a LinkedHashSet to keep the order. + final Set suggestedLocations = new LinkedHashSet<>(); for (int i = 0; i < arr.length(); i++) { - final JSONObject jsonObject = arr.getJSONObject(i); - - // Skip if the result is not in GB - if (!jsonObject.getJSONObject("address").getString("countryRegionIso2").equals(REGION)) { - continue; + final JSONObject jsonObject = arr.getJSONObject(i).getJSONObject("address"); + final SuggestedLocation suggestedLocation = createSuggestedLocation(jsonObject); + suggestedLocations.add(suggestedLocation); + } + // Now convert the suggestions by Bing to actual locations, to obtain geo information. + int i = 0; + for (final SuggestedLocation suggestedLocation : suggestedLocations) { + final JSONObject jsonObject = obtainLocation(suggestedLocation); + if (jsonObject != null) { + final SearchSuggestion sug = createSuggestion(query, i, jsonObject); + sugs.merge(sug.getDescription(), sug, (a, b) -> a.getScore() > b.getScore() ? a : b); + i++; } - - final SearchSuggestion sug = createSuggestion(query, i, jsonObject); - sugs.merge(sug.getDescription(), sug, (a, b) -> a.getScore() > b.getScore() ? a : b); } } + private SuggestedLocation createSuggestedLocation(final JSONObject jsonObject) { + return new SuggestedLocation(jsonObject.optString("locality"), + jsonObject.optString("adminDistrict2"), + jsonObject.optString("addressLine"), + jsonObject.optString("formattedAddress")); + } + + private JSONObject obtainLocation(final SuggestedLocation location) { + final String url = String.format(BING_lOCATIONS_ENDPOINT, apiKey, location.toUrlParameters()); + final HttpResponse json = Unirest.get(url).asJson(); + final JSONObject body = json.getBody().getObject(); + final JSONArray arr = body.getJSONArray("resourceSets").getJSONObject(0).getJSONArray("resources"); + return arr.length() == 0 ? null : arr.getJSONObject(0); + } + private SearchSuggestion createSuggestion(final String query, final int idx, final JSONObject jsonObject) { final String id = "id-" + query + "-" + idx; final String displayText = jsonObject.getString("name"); - final double score = getScoreFromConfidence(jsonObject.getString("confidence")); + // While there is a confidence bit, as we use a 2nd call we can't really use it + /// (the confidence is always based on input for that second call, which isn't user input) + // Instead, assume that Bing knows what it's doing with the autosuggest and score earlier records higher + final double score = 90D - idx; final SearchSuggestionType type = determineType(jsonObject.getString("entityType")); final JSONArray bbox = jsonObject.getJSONArray("bbox"); final String wktBbox = "POLYGON((" @@ -118,19 +165,6 @@ private SearchSuggestion createSuggestion(final String query, final int idx, fin return suggestion; } - private static double getScoreFromConfidence(final String confidence) { - switch (confidence) { - case "High": - return 90D; - case "Medium": - return 50D; - case "Low": - return 30D; - default: - return 15D; - } - } - private static SearchSuggestionType determineType(final String type) { SearchSuggestionType suggestionType; @@ -162,4 +196,58 @@ private static SearchSuggestionType determineType(final String type) { return suggestionType; } + + private static class SuggestedLocation { + + private final String locality; + private final String adminDistrict; + private final String addressLine; + private final String formattedAddress; + + SuggestedLocation(final String locality, final String adminDistrict, final String addressLine, final String formattedAddress) { + this.locality = locality; + this.adminDistrict = adminDistrict; + this.addressLine = addressLine; + this.formattedAddress = formattedAddress; + } + + @Override + public int hashCode() { + return Objects.hash(addressLine, adminDistrict, formattedAddress, locality); + } + + @Override + public boolean equals(final Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + final SuggestedLocation other = (SuggestedLocation) obj; + return Objects.equals(addressLine, other.addressLine) && Objects.equals(adminDistrict, other.adminDistrict) + && Objects.equals(formattedAddress, other.formattedAddress) && Objects.equals(locality, other.locality); + } + + String toUrlParameters() { + final Map parameters = new HashMap<>(); + if (locality != null) { + parameters.put("locality", locality); + } + if (adminDistrict != null) { + parameters.put("adminDistrict", adminDistrict); + } + if (addressLine != null) { + parameters.put("addressLine", addressLine); + } + if (formattedAddress != null) { + parameters.put("query", formattedAddress); + } + return parameters.isEmpty() + ? "" + : "&" + parameters.entrySet().stream() + .map(e -> e.getKey() + "=" + e.getValue()) + .collect(Collectors.joining("&")); + } + } } diff --git a/search-service-bing-geocoder/src/test/java/nl/aerius/search/tasks/BingSearchServiceTest.java b/search-service-bing-geocoder/src/test/java/nl/aerius/search/tasks/BingSearchServiceTest.java index 20dd8eb..cafdae0 100644 --- a/search-service-bing-geocoder/src/test/java/nl/aerius/search/tasks/BingSearchServiceTest.java +++ b/search-service-bing-geocoder/src/test/java/nl/aerius/search/tasks/BingSearchServiceTest.java @@ -40,10 +40,10 @@ void testWorksAtAll() { return; } - final Single result = delegator.retrieveSearchResults("utrecht"); + final Single result = delegator.retrieveSearchResults("edin"); final SearchTaskResult suggestions = result.blockingGet(); - assertEquals(10, suggestions.getSuggestions().size(), "Expected 10 results for 'utrecht'"); + assertEquals(7, suggestions.getSuggestions().size(), "Expected number of results for 'edin' (should include 'edinburgh')"); } } From c480aea5971beab52ce3b8d87d15db7ae2ca0665 Mon Sep 17 00:00:00 2001 From: Bert Scholten Date: Fri, 1 Dec 2023 16:15:50 +0100 Subject: [PATCH 2/2] Adjusted second call to include name information The name does not really fit in the locations-with-adress endpoint (when using it along with countryregion, you'll get just United Kingdom as a result...), so have to use the locations-with-query endpoint (which is the same, but has different query parameters, not the cleanest endpoint definition ever). Does mean more results are returned, so reduced number of max results. --- .../search/tasks/BingSearchService.java | 53 +++++++++++-------- .../search/tasks/BingSearchServiceTest.java | 2 +- 2 files changed, 32 insertions(+), 23 deletions(-) diff --git a/search-service-bing-geocoder/src/main/java/nl/aerius/search/tasks/BingSearchService.java b/search-service-bing-geocoder/src/main/java/nl/aerius/search/tasks/BingSearchService.java index a4e5257..cbfc254 100644 --- a/search-service-bing-geocoder/src/main/java/nl/aerius/search/tasks/BingSearchService.java +++ b/search-service-bing-geocoder/src/main/java/nl/aerius/search/tasks/BingSearchService.java @@ -58,11 +58,10 @@ public class BingSearchService implements SearchTaskService { + "&includeEntityTypes=Address,Place" + "&c=" + CULTURE + "&userMapView=" + BNG_BOUNDS - + "&maxResults=10"; - private static final String BING_lOCATIONS_ENDPOINT = "https://dev.virtualearth.net/REST/v1/Locations?" + + "&maxResults=5"; + private static final String BING_LOCATIONS_ENDPOINT = "https://dev.virtualearth.net/REST/v1/Locations?" + "key=%s" + "&userRegion=" + REGION - + "&countryRegion=" + REGION + "&c=" + CULTURE + "&userMapView=" + BNG_BOUNDS + "&maxResults=1" @@ -97,18 +96,13 @@ private void retrieveSuggestions(final String query, final Map json = Unirest.get(url).asJson(); - final JSONObject body = json.getBody().getObject(); - - final JSONArray arr = body - .getJSONArray("resourceSets").getJSONObject(0) - .getJSONArray("resources").getJSONObject(0) + final JSONArray arr = obtainResources(url).getJSONObject(0) .getJSONArray("value"); // As the list of suggestions can contain duplicates when we translate them to actual locations, filter those out by using a set // Use a LinkedHashSet to keep the order. final Set suggestedLocations = new LinkedHashSet<>(); for (int i = 0; i < arr.length(); i++) { - final JSONObject jsonObject = arr.getJSONObject(i).getJSONObject("address"); + final JSONObject jsonObject = arr.getJSONObject(i); final SuggestedLocation suggestedLocation = createSuggestedLocation(jsonObject); suggestedLocations.add(suggestedLocation); } @@ -124,18 +118,29 @@ private void retrieveSuggestions(final String query, final Map json = Unirest.get(url).asJson(); + final JSONObject body = json.getBody().getObject(); + return body.getJSONArray("resourceSets").getJSONObject(0).getJSONArray("resources"); + } + private SuggestedLocation createSuggestedLocation(final JSONObject jsonObject) { - return new SuggestedLocation(jsonObject.optString("locality"), - jsonObject.optString("adminDistrict2"), - jsonObject.optString("addressLine"), - jsonObject.optString("formattedAddress")); + final JSONObject addressObject = jsonObject.getJSONObject("address"); + return new SuggestedLocation(jsonObject.optString("name"), + addressObject.optString("locality"), + addressObject.optString("adminDistrict2"), + addressObject.optString("addressLine"), + addressObject.optString("formattedAddress")); } private JSONObject obtainLocation(final SuggestedLocation location) { - final String url = String.format(BING_lOCATIONS_ENDPOINT, apiKey, location.toUrlParameters()); - final HttpResponse json = Unirest.get(url).asJson(); - final JSONObject body = json.getBody().getObject(); - final JSONArray arr = body.getJSONArray("resourceSets").getJSONObject(0).getJSONArray("resources"); + final String url; + if (location.name != null && !location.name.isEmpty()) { + url = String.format(BING_LOCATIONS_ENDPOINT, apiKey, "&query=" + location.name); + } else { + url = String.format(BING_LOCATIONS_ENDPOINT, apiKey, location.toAddressUrlParameters()); + } + final JSONArray arr = obtainResources(url); return arr.length() == 0 ? null : arr.getJSONObject(0); } @@ -199,12 +204,14 @@ private static SearchSuggestionType determineType(final String type) { private static class SuggestedLocation { + private final String name; private final String locality; private final String adminDistrict; private final String addressLine; private final String formattedAddress; - SuggestedLocation(final String locality, final String adminDistrict, final String addressLine, final String formattedAddress) { + SuggestedLocation(final String name, final String locality, final String adminDistrict, final String addressLine, final String formattedAddress) { + this.name = name; this.locality = locality; this.adminDistrict = adminDistrict; this.addressLine = addressLine; @@ -213,7 +220,7 @@ private static class SuggestedLocation { @Override public int hashCode() { - return Objects.hash(addressLine, adminDistrict, formattedAddress, locality); + return Objects.hash(addressLine, adminDistrict, formattedAddress, locality, name); } @Override @@ -226,11 +233,13 @@ public boolean equals(final Object obj) { return false; final SuggestedLocation other = (SuggestedLocation) obj; return Objects.equals(addressLine, other.addressLine) && Objects.equals(adminDistrict, other.adminDistrict) - && Objects.equals(formattedAddress, other.formattedAddress) && Objects.equals(locality, other.locality); + && Objects.equals(formattedAddress, other.formattedAddress) && Objects.equals(locality, other.locality) + && Objects.equals(name, other.name); } - String toUrlParameters() { + String toAddressUrlParameters() { final Map parameters = new HashMap<>(); + parameters.put("countryRegion", REGION); if (locality != null) { parameters.put("locality", locality); } diff --git a/search-service-bing-geocoder/src/test/java/nl/aerius/search/tasks/BingSearchServiceTest.java b/search-service-bing-geocoder/src/test/java/nl/aerius/search/tasks/BingSearchServiceTest.java index cafdae0..f3af03d 100644 --- a/search-service-bing-geocoder/src/test/java/nl/aerius/search/tasks/BingSearchServiceTest.java +++ b/search-service-bing-geocoder/src/test/java/nl/aerius/search/tasks/BingSearchServiceTest.java @@ -44,6 +44,6 @@ void testWorksAtAll() { final SearchTaskResult suggestions = result.blockingGet(); - assertEquals(7, suggestions.getSuggestions().size(), "Expected number of results for 'edin' (should include 'edinburgh')"); + assertEquals(5, suggestions.getSuggestions().size(), "Expected number of results for 'edin' (should include 'edinburgh')"); } }