Rdcc 7010 (#649)

* initial commit

* log in to exception table the mismatched object Ids from idam

* repository changes to be pushed

* repository changes to be pushed

* unit test cases

* mismatched object ids in to exception table

* integration test case fly way

* integration test case fly way

* integration test case

* send Email for mismatched ObjectIds

* email template

* test cases ...

* test cases ...

* test cases ...

* test cases ...

* test cases ...

* check style fix

* check style fix

* test case fix

* error description message fix

src/integrationTest/java/uk/gov/hmcts/reform/judicialapi/elinks/ElinksEndToEndIntegrationPartialTest.java

@@ -281,7 +281,7 @@ void test_elinks_end_to_end_success_scenario_with_partial_success_return_status_
         validateSidamPublish();
 
         List<ElinkDataExceptionRecords> elinksException = elinkDataExceptionRepository.findAll();
-        assertThat(elinksException).hasSize(4);
+        assertThat(elinksException).hasSize(5);
 
     }
 

src/integrationTest/java/uk/gov/hmcts/reform/judicialapi/elinks/ElinksEndToEndIntegrationTest.java

@@ -268,7 +268,7 @@ void test_elinks_end_to_end_success_scenario_with_return_status_200()
         validateSidamPublish();
 
         List<ElinkDataExceptionRecords> elinksException = elinkDataExceptionRepository.findAll();
-        assertThat(elinksException).isEmpty();
+        assertThat(elinksException).isNotEmpty();
 
     }
 
@@ -385,7 +385,7 @@ private void validateSidamPublish() {
         assertThat(publishSidamIdsResponse.get("publishing_status")).isNotNull();
 
         List<ElinkDataExceptionRecords> elinksException = elinkDataExceptionRepository.findAll();
-        assertThat(elinksException).isEmpty();
+        assertThat(elinksException).isNotEmpty();
     }
 
     private void validateElasticSearch(List<DataloadSchedulerJob> audits) {

src/integrationTest/java/uk/gov/hmcts/reform/judicialapi/elinks/ElinksFailedForExtraBaseLocationIdEndToEndIntegrationTest.java

@@ -226,7 +226,7 @@ private void validateSidamPublish() {
         assertThat(publishSidamIdsResponse.get("publishing_status")).isNotNull();
 
         List<ElinkDataExceptionRecords> elinksException = elinkDataExceptionRepository.findAll();
-        assertEquals(18,elinksException.size());
+        assertEquals(20,elinksException.size());
     }
 
     private void validateBaseLocationApi(List<ElinkDataSchedularAudit> elinksAudit) {

src/integrationTest/resources/db/testmigration/V1_84__elinks_alter_dataload_exception_records.sql

@@ -0,0 +1,2 @@
+ALTER TABLE dbjudicialdata.dataload_exception_records
+ALTER COLUMN row_id TYPE VARCHAR(256);

src/main/java/uk/gov/hmcts/reform/judicialapi/elinks/repository/ProfileRepository.java

@@ -96,4 +96,12 @@ List<UserSearchResponseWrapper> findBySearchForString(String searchString, Strin
     Page<UserProfile> fetchUserProfileByServiceNames(Set<String> ccdServiceCode,
                                                      List<String> ticketCode, Pageable pageable);
 
+
+    @Query(value = "select distinct per.objectId "
+            + "from judicialUserProfile per "
+            + "where (per.objectId != '' and per.objectId is not null)")
+    List<String> fetchObjectId();
+
+
+
 }

src/main/java/uk/gov/hmcts/reform/judicialapi/elinks/service/impl/ElinksPeopleServiceImpl.java

@@ -10,16 +10,13 @@
 import org.springframework.format.annotation.DateTimeFormat;
 import org.springframework.http.HttpStatus;
 import org.springframework.http.ResponseEntity;
-import org.springframework.jdbc.core.JdbcTemplate;
 import org.springframework.stereotype.Service;
-import uk.gov.hmcts.reform.judicialapi.elinks.configuration.ElinkEmailConfiguration;
 import uk.gov.hmcts.reform.judicialapi.elinks.controller.request.AppointmentsRequest;
 import uk.gov.hmcts.reform.judicialapi.elinks.controller.request.AuthorisationsRequest;
 import uk.gov.hmcts.reform.judicialapi.elinks.controller.request.PeopleRequest;
 import uk.gov.hmcts.reform.judicialapi.elinks.controller.request.ResultsRequest;
 import uk.gov.hmcts.reform.judicialapi.elinks.controller.request.RoleRequest;
 import uk.gov.hmcts.reform.judicialapi.elinks.domain.Appointment;
-import uk.gov.hmcts.reform.judicialapi.elinks.domain.ElinkDataExceptionRecords;
 import uk.gov.hmcts.reform.judicialapi.elinks.domain.JudicialRoleType;
 import uk.gov.hmcts.reform.judicialapi.elinks.domain.Location;
 import uk.gov.hmcts.reform.judicialapi.elinks.domain.UserProfile;
@@ -29,34 +26,28 @@
 import uk.gov.hmcts.reform.judicialapi.elinks.repository.AuthorisationsRepository;
 import uk.gov.hmcts.reform.judicialapi.elinks.repository.BaseLocationRepository;
 import uk.gov.hmcts.reform.judicialapi.elinks.repository.DataloadSchedularAuditRepository;
-import uk.gov.hmcts.reform.judicialapi.elinks.repository.ElinkDataExceptionRepository;
 import uk.gov.hmcts.reform.judicialapi.elinks.repository.JrdRegionMappingRepository;
 import uk.gov.hmcts.reform.judicialapi.elinks.repository.JudicialRoleTypeRepository;
 import uk.gov.hmcts.reform.judicialapi.elinks.repository.LocationMapppingRepository;
 import uk.gov.hmcts.reform.judicialapi.elinks.repository.LocationRepository;
 import uk.gov.hmcts.reform.judicialapi.elinks.repository.ProfileRepository;
 import uk.gov.hmcts.reform.judicialapi.elinks.response.ElinkPeopleWrapperResponse;
 import uk.gov.hmcts.reform.judicialapi.elinks.service.ElinksPeopleService;
-import uk.gov.hmcts.reform.judicialapi.elinks.service.IEmailService;
-import uk.gov.hmcts.reform.judicialapi.elinks.service.dto.Email;
 import uk.gov.hmcts.reform.judicialapi.elinks.util.CommonUtil;
 import uk.gov.hmcts.reform.judicialapi.elinks.util.ElinkDataExceptionHelper;
 import uk.gov.hmcts.reform.judicialapi.elinks.util.ElinkDataIngestionSchedularAudit;
-import uk.gov.hmcts.reform.judicialapi.elinks.util.EmailTemplate;
 import uk.gov.hmcts.reform.judicialapi.elinks.util.RefDataElinksConstants;
+import uk.gov.hmcts.reform.judicialapi.elinks.util.SendEmail;
 import uk.gov.hmcts.reform.judicialapi.util.JsonFeignResponseUtil;
 
 import java.time.LocalDate;
 import java.time.LocalDateTime;
 import java.time.format.DateTimeFormatter;
 import java.util.ArrayList;
 import java.util.HashMap;
-import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
-import java.util.Set;
-import java.util.stream.Collectors;
 
 import static java.time.LocalDateTime.now;
 import static java.util.Objects.isNull;
@@ -68,9 +59,7 @@
 import static uk.gov.hmcts.reform.judicialapi.elinks.util.RefDataElinksConstants.BASE_LOCATION;
 import static uk.gov.hmcts.reform.judicialapi.elinks.util.RefDataElinksConstants.BASE_LOCATION_ID;
 import static uk.gov.hmcts.reform.judicialapi.elinks.util.RefDataElinksConstants.CFTREGIONIDFAILURE;
-import static uk.gov.hmcts.reform.judicialapi.elinks.util.RefDataElinksConstants.CONTENT_TYPE_HTML;
 import static uk.gov.hmcts.reform.judicialapi.elinks.util.RefDataElinksConstants.DATA_UPDATE_ERROR;
-import static uk.gov.hmcts.reform.judicialapi.elinks.util.RefDataElinksConstants.DATE_PATTERN;
 import static uk.gov.hmcts.reform.judicialapi.elinks.util.RefDataElinksConstants.ELINKS_ACCESS_ERROR;
 import static uk.gov.hmcts.reform.judicialapi.elinks.util.RefDataElinksConstants.ELINKS_ERROR_RESPONSE_BAD_REQUEST;
 import static uk.gov.hmcts.reform.judicialapi.elinks.util.RefDataElinksConstants.ELINKS_ERROR_RESPONSE_FORBIDDEN;
@@ -120,7 +109,7 @@ public class ElinksPeopleServiceImpl implements ElinksPeopleService {
     private AuthorisationsRepository authorisationsRepository;
 
     @Autowired
-    ElinkEmailConfiguration emailConfiguration;
+    SendEmail sendEmail;
 
     @Autowired
     private ProfileRepository profileRepository;
@@ -145,9 +134,6 @@ public class ElinksPeopleServiceImpl implements ElinksPeopleService {
 
     @Autowired ElinksPeopleDeleteServiceimpl elinksPeopleDeleteServiceimpl;
 
-    @Autowired
-    JdbcTemplate jdbcTemplate;
-
     @Autowired
     CommonUtil commonUtil;
 
@@ -156,12 +142,6 @@ public class ElinksPeopleServiceImpl implements ElinksPeopleService {
     private Map<String,UserProfile> userProfileCache = new HashMap<String,UserProfile>();
     private List<UserProfile> userProfilesSnapshot = new ArrayList<>();
 
-    @Autowired
-    ElinkDataExceptionRepository elinkDataExceptionRepository;
-
-    @Value("${logging-component-name}")
-    private String logComponentName;
-
     @Value("${elinks.people.lastUpdated}")
     @DateTimeFormat(pattern = "yyyy-MM-dd")
     private String lastUpdated;
@@ -187,11 +167,6 @@ public class ElinksPeopleServiceImpl implements ElinksPeopleService {
     @Value("${elinks.people.includePreviousAppointments}")
     private String includePreviousAppointments;
 
-    @Autowired
-    EmailTemplate emailTemplate;
-
-    @Autowired
-    IEmailService emailService;
 
     private final Map<String, String> emailConfigMapping = Map.of(LOCATION_ID, REGION,
             BASE_LOCATION_ID, BASE_LOCATION);
@@ -248,12 +223,8 @@ public ResponseEntity<ElinkPeopleWrapperResponse> updatePeople() {
             auditStatus(schedulerStartTime, RefDataElinksConstants.JobStatus.FAILED.getStatus());
             throw ex;
         }
-        List<ElinkDataExceptionRecords> list = elinkDataExceptionRepository
-                .findBySchedulerStartTime(schedulerStartTime);
-        if (!list.isEmpty()) {
-            sendEmail(new HashSet<>(list), "baselocation",
-                    LocalDate.now().format(DateTimeFormatter.ofPattern(DATE_PATTERN)));
-        }
+        sendEmail.sendEmail(schedulerStartTime);
+
         if (partialSuccessFlag) {
             status = RefDataElinksConstants.JobStatus.PARTIAL_SUCCESS.getStatus();
         }
@@ -269,48 +240,8 @@ public ResponseEntity<ElinkPeopleWrapperResponse> updatePeople() {
                 .body(response);
     }
 
-    private void sendEmail(LocalDateTime schedulerStartTime) {
-        List<ElinkDataExceptionRecords> list = elinkDataExceptionRepository
-                .findBySchedulerStartTime(schedulerStartTime);
 
-        Map<String, List<ElinkDataExceptionRecords>> map = list
-                .stream()
-                .collect(Collectors.groupingBy(ElinkDataExceptionRecords::getFieldInError));
 
-        if (map.containsKey(BASE_LOCATION_ID)) {
-            sendEmail(new HashSet<>(map.get(BASE_LOCATION_ID)), "baselocation",
-                    LocalDate.now().format(DateTimeFormatter.ofPattern(DATE_PATTERN)));
-        }
-        if (map.containsKey(LOCATION)) {
-            sendEmail(new HashSet<>(map.get(LOCATION)), "location",
-                    LocalDate.now().format(DateTimeFormatter.ofPattern(DATE_PATTERN)));
-        }
-        if (map.containsKey(APPOINTMENTID)) {
-            sendEmail(new HashSet<>(map.get(APPOINTMENTID)), "appointment",
-                    LocalDate.now().format(DateTimeFormatter.ofPattern(DATE_PATTERN)));
-        }
-        if (map.containsKey(USER_PROFILE)) {
-            sendEmail(new HashSet<>(map.get(USER_PROFILE)), "userprofile",
-                LocalDate.now().format(DateTimeFormatter.ofPattern(DATE_PATTERN)));
-        }
-    }
-
-    public int sendEmail(Set<ElinkDataExceptionRecords> data, String type, Object... params) {
-        log.info("{} : send Email",logComponentName);
-        ElinkEmailConfiguration.MailTypeConfig config = emailConfiguration.getMailTypes()
-                .get(type);
-        if (config != null && config.isEnabled()) {
-            Email email = Email.builder()
-                    .contentType(CONTENT_TYPE_HTML)
-                    .from(config.getFrom())
-                    .to(config.getTo())
-                    .subject(String.format(config.getSubject(), params))
-                    .messageBody(emailTemplate.getEmailBody(config.getTemplate(), Map.of("resultsRequest", data)))
-                    .build();
-            return emailService.sendEmail(email);
-        }
-        return -1;
-    }
 
     private void auditStatus(LocalDateTime schedulerStartTime, String status) {
         elinkDataIngestionSchedularAudit.auditSchedulerStatus(JUDICIAL_REF_DATA_ELINKS,
@@ -421,7 +352,7 @@ private void saveRoleDetails(String personalCode, List<RoleRequest> judiciaryRol
 
 
     private boolean saveUserProfile(ResultsRequest resultsRequest,LocalDateTime schedulerStartTime, int pageValue) {
-        
+
         if (validateUserProfile(resultsRequest, schedulerStartTime,pageValue)) {
             try {
                 LocalDateTime createdOn = null;

src/main/java/uk/gov/hmcts/reform/judicialapi/elinks/service/impl/IdamElasticSearchServiceImpl.java

@@ -16,11 +16,14 @@
 import uk.gov.hmcts.reform.judicialapi.elinks.configuration.IdamTokenConfigProperties;
 import uk.gov.hmcts.reform.judicialapi.elinks.exception.ElinksException;
 import uk.gov.hmcts.reform.judicialapi.elinks.feign.IdamFeignClient;
+import uk.gov.hmcts.reform.judicialapi.elinks.repository.ProfileRepository;
 import uk.gov.hmcts.reform.judicialapi.elinks.response.IdamOpenIdTokenResponse;
 import uk.gov.hmcts.reform.judicialapi.elinks.response.IdamResponse;
 import uk.gov.hmcts.reform.judicialapi.elinks.service.IdamElasticSearchService;
+import uk.gov.hmcts.reform.judicialapi.elinks.util.ElinkDataExceptionHelper;
 import uk.gov.hmcts.reform.judicialapi.elinks.util.ElinkDataIngestionSchedularAudit;
 import uk.gov.hmcts.reform.judicialapi.elinks.util.RefDataElinksConstants;
+import uk.gov.hmcts.reform.judicialapi.elinks.util.SendEmail;
 import uk.gov.hmcts.reform.judicialapi.elinks.util.SqlConstants;
 import uk.gov.hmcts.reform.judicialapi.util.JsonFeignResponseUtil;
 
@@ -36,13 +39,16 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.stream.Collectors;
 
 import static java.time.LocalDateTime.now;
 import static java.util.Objects.nonNull;
 import static uk.gov.hmcts.reform.judicialapi.elinks.util.RefDataElinksConstants.ELASTICSEARCH;
 import static uk.gov.hmcts.reform.judicialapi.elinks.util.RefDataElinksConstants.IDAM_ERROR_MESSAGE;
 import static uk.gov.hmcts.reform.judicialapi.elinks.util.RefDataElinksConstants.IDAM_TOKEN_ERROR_MESSAGE;
 import static uk.gov.hmcts.reform.judicialapi.elinks.util.RefDataElinksConstants.JUDICIAL_REF_DATA_ELINKS;
+import static uk.gov.hmcts.reform.judicialapi.elinks.util.RefDataElinksConstants.OBJECT_ID;
+import static uk.gov.hmcts.reform.judicialapi.elinks.util.RefDataElinksConstants.USER_PROFILE;
 
 @Slf4j
 @Component
@@ -66,9 +72,21 @@ public class IdamElasticSearchServiceImpl implements IdamElasticSearchService {
     @Autowired
     JdbcTemplate jdbcTemplate;
 
+    @Autowired
+    SendEmail sendEmail;
+
+    @Autowired
+    private ProfileRepository userProfileRepository;
+
+    @Autowired
+    ElinkDataExceptionHelper elinkDataExceptionHelper;
+
     @Autowired
     ElinkDataIngestionSchedularAudit elinkDataIngestionSchedularAudit;
 
+    @Value("${elinks.people.page}")
+    String page;
+
     @Override
     public String getIdamBearerToken(LocalDateTime schedulerStartTime) {
         IdamOpenIdTokenResponse idamOpenIdTokenResponse = null;
@@ -163,6 +181,10 @@ public ResponseEntity<Object> getIdamElasticSearchSyncFeed() {
             count++;
             log.debug("{}:: batch count :: ", count);
         } while (totalCount > 0 && recordsPerPage * count < totalCount);
+
+        validateObjectIds(judicialUsers,schedulerStartTime);
+        sendEmail.sendEmail(schedulerStartTime);
+
         updateSidamIds(judicialUsers);
         elinkDataIngestionSchedularAudit.auditSchedulerStatus(JUDICIAL_REF_DATA_ELINKS,
             schedulerStartTime,
@@ -174,6 +196,34 @@ public ResponseEntity<Object> getIdamElasticSearchSyncFeed() {
                 .body(judicialUsers);
     }
 
+    public void validateObjectIds(Set<IdamResponse> sidamUsers,LocalDateTime schedulerStartTime) {
+
+        Map<String,String> sidamObjectId = new HashMap<>();
+
+        sidamUsers.stream().filter(user -> nonNull(user.getSsoId())).forEach(s ->
+                sidamObjectId.put(s.getSsoId(), s.getId()));
+
+        List<String> jrdObjectIdsList = userProfileRepository.fetchObjectId();
+
+        Map<String,String> filteredObjectIds = sidamObjectId.entrySet()
+                .stream()
+                .filter(entry -> !jrdObjectIdsList.contains(entry.getKey()))
+                .collect(Collectors.toUnmodifiableMap(Map.Entry::getKey, Map.Entry::getValue));
+
+        String errorDescription = "Object ID, received from Idam, is not present in Judicial Reference Data";
+        int pageValue = Integer.parseInt(page);
+
+
+        if (!filteredObjectIds.isEmpty()) {
+            for (var entry : filteredObjectIds.entrySet()) {
+                elinkDataExceptionHelper.auditException(JUDICIAL_REF_DATA_ELINKS,
+                        schedulerStartTime,
+                        entry.getKey(),
+                        OBJECT_ID, errorDescription, USER_PROFILE, entry.getValue(),pageValue);
+            }
+        }
+    }
+
     private void logIdamResponses(Response response) {
         log.debug("Logging Response from IDAM");
         if (response != null) {

src/main/java/uk/gov/hmcts/reform/judicialapi/elinks/util/RefDataElinksConstants.java

@@ -70,6 +70,8 @@ private RefDataElinksConstants() {
 
     public static final String BASE_LOCATION_ID = "base_location_id";
 
+    public static final String OBJECT_ID = "object_id";
+
     public static final String LOCATION_ID = "region_id";
 
     public static final String LOCATION = "Location";