diff --git a/Docs/dss-dashboard/DSS Analytics Dashboard YAML Spec 1.0.0.yaml b/Docs/dss-dashboard/DSS Analytics Dashboard YAML Spec 1.0.0.yaml new file mode 100644 index 000000000..a254b6c90 --- /dev/null +++ b/Docs/dss-dashboard/DSS Analytics Dashboard YAML Spec 1.0.0.yaml @@ -0,0 +1,369 @@ + +swagger: '2.0' +info: + title: DSS Analytics APIs + description: | + APIs for DSS Analytics Dashboard will mainly consist of 2 APIs. + The one which is used to get Configuration based on which the Dashboard has to be painted. + The other one will be the API which gets the data which is needed for the Visualization to paint different Graphs. + version: "1.0.0" +# the domain of the service, only for example +host: 'egov-micro-dev.egovernments.org' +# array of all schemes that the API supports, preferred https +schemes: + - http +# will be prefixed to all paths +basePath: '/dashboard/[APIEndPoint]' +produces: + - application/json + +paths: + /getDashboardConfig/{dashboardId}: + get: + summary: API to get the Dashboard Configuration which gives the Visualizations and Layout to paint + description: + This endpoint will pass the Dashboard ID from the URL to this API and will get the Dashboard Configuration from + the backend which will have the details of the Visualization Layouts which are to be designed in this Dashboard. + tags: + - Dashboard + parameters: + - name: dashboardId + in: path + description: ID of the Dashboard + required: true + type: string + responses: + '200': + description: Response for Success State of the API + schema: + $ref: '#/definitions/DashboardConfigResponse' + '400': + description: Invalid input. + schema: + $ref: '#/definitions/ErrorRes' + + /getChartV2: + post: + summary: Get Chart API will be responding with the Data for each visualization based on the Chart ID which has been sent + description: + Based on the Chart ID and the Visualization Type and the Request Dates and Filters, the API will aggregate the data and give the data which needs to be painted on Visualization Charts + tags: + - Order + parameters: + - in: body + name: order + required: true + description: This request object contains the Visualization for which the data has to be served. + schema: + $ref: '#/definitions/DashboardRequest' + responses: + '200': + description: Response for Success State of the API + schema: + $ref: '#/definitions/DashboardResponse' + '400': + description: Invalid input. + schema: + $ref: '#/definitions/ErrorRes' + + +definitions: + Chart: + type: object + properties: + id: + type: integer + format: int64 + description: This field holds the Unique Identifier of the Charts + name: + type: string + description: Name of the Chart + code: + type: string + description: Unique Code to recognize the Chart + chartType: + type: string + description: Type of the Chart or Graph which will be painted on the UI for this partifular Chart Data is represented here. + format: enum + - PIE + - BAR + - METRIC + - LINE + - TABLE + filter: + type: object + description: A map of String and and String to hold the Filters which are applied on this chart + additionalProperties: + type: string + + Dimension: + type: object + properties: + height: + type: integer + description: Describes the Height of the Visualization + width: + type: integer + description: Describes the Width of the Visualization + + Plot: + type: object + properties: + label: + type: string + description: Label which has to be printed in the Visualization Charts + name: + type: string + description: Name of the legends which are to be used in the Charts + value: + type: integer + format: int64 + description: Holds the value of plot response + symbol: + type: string + description: Symbol that has to be used with the Value. Amount or Value or Percentage etc., + + Data: + type: object + properties: + headerName: + type: string + description: Name of the header when the grouping of all the plots are done + headerValue: + type: integer + format: int64 + description: Total Value of all the plots are put as Header Value + headerSymbol: + type: string + description: Symbol which has to be used to represent + plots: + type: array + description: Symbol that has to be used with the Value. Amount or Value or Percentage etc., + items: + $ref: "#/definitions/Plot" + + Visualization: + type: object + properties: + id: + type: integer + description: Unique Identifier for a Visualization + name: + type: string + description: Name of the Visualization + dimensions: + $ref: "#/definitions/Dimension" + vizType: + type: string + description: Type of Visualization which groups the type of different charts in specific + charts: + type: array + items: + $ref: "#/definitions/Chart" + + Tab: + type: object + properties: + row: + type: integer + description: This defines the row number in which this Visualization will have to appear on UI + name: + type: string + description: Name of the tab on which the visualization has to appear is defined here + vizArray: + type: array + items: + $ref: "#/definitions/Visualization" + + Dashboard: + type: object + properties: + name: + type: string + description: Name of the Dashboard + id: + type: string + description: Unique ID of the Dashboard + isActive: + type: boolean + description: Active / Inactive Status of the Dashboard + style: + type: string + description: Type of the Style which will be represented + format: enum + - LINEAR + - TABBED + visualizations: + type: array + items: + $ref: "#/definitions/Tab" + + RequestDate: + type: object + properties: + startDate: + type: string + description: Request Date Starting Date field + endDate: + type: string + description: Request Date Ending Date Field + + AggregationRequest: + type: object + properties: + visualizationType: + type: string + description: Request mentions the type of Visualization which it needs the data for + format: enum + - PIE + - BAR + - METRIC + - LINE + - TABLE + visualizationCode: + type: string + description: Code of the Visualization for which the data has to be obtained. + queryType: + type: string + description: Type of Query which has to be used to get the data for this visualization + filter: + type: object + description: A map of String and and String to hold the Filters which are applied on this chart + additionalProperties: + type: string + moduleLevel: + type: string + description: Which Module is being represented is to be added here. + format: enum + - PT + - TL + - PGR + - HOME_REVENUE + - HOME_SERVICES + requestDate: + type: object + $ref: "#/definitions/RequestDate" + + DashboardRequest: + type: object + properties: + header: + type: object + description: A map of String and and String to hold the Header Filters which are applied on this dashboard + additionalProperties: + type: string + aggregationRequest: + $ref: "#/definitions/AggregationRequest" + + DashboardConfigResponse: + type: object + properties: + statusInfo: + type: object + description: Status Info + $ref: "#/definitions/StatusInfo" + responseData: + type: array + items: + $ref: "#/definitions/Dashboard" + + DashboardResponse: + type: object + properties: + statusInfo: + type: object + description: Status Info + $ref: "#/definitions/StatusInfo" + responseData: + type: array + items: + $ref: "#/definitions/ChartApiResponse" + + ChartApiResponse: + type: object + properties: + chartType: + type: string + description: Type of Chart which are to be used to represent the data on Dashboard + maxLength: 128 + chartFormat: + type: string + description: Format specifies as which library to be used to paint the visualizations + maxLength: 32 + drillDownChartId: + type: string + description: Chart ID which is to be used for Drill Downs + filter: + type: object + description: A map of String and and String to hold the Filters which are applied on this chart + additionalProperties: + type: string + data: + type: array + items: + $ref: "#/definitions/Data" + + SuccessResponse: + type: object + properties: + status: + $ref: "#/definitions/StatusInfo" + + StatusInfo: + type: object + readOnly: true + description: Status Information object will hold the run time status and the live response obtained from the server for the respective API Call. + properties: + statusCode: + type: string + description: unique API ID + maxLength: 128 + statusMessage: + type: string + description: API version + maxLength: 32 + errorCode: + type: integer + format: int64 + description: response time in epoch + errorMessage: + type: string + description: unique response message id (UUID) - will usually be the correlation id from the server + maxLength: 256 + + Error: + type: object + description: Error object will be returned as a part of reponse body in conjunction with ResponseInfo as part of ErrorResponse whenever the request processing status in the ResponseInfo is FAILED. HTTP return in this scenario will usually be HTTP 400. + properties: + code: + type: string + description: Error Code will be module specific error label/code to identiffy the error. All modules should also publish the Error codes with their specific localized values in localization service to ensure clients can print locale specific error messages. Example for error code would be User.NotFound to indicate User Not Found by User/Authentication service. All services must declare their possible Error Codes with brief description in the error response section of their API path. + message: + type: string + description: English locale message of the error code. Clients should make a separate call to get the other locale description if configured with the service. + description: + type: string + description: Optional long description of the error to help clients take remedial action. This will not be available as part of localization service. + params: + type: array + description: Some error messages may carry replaceable fields (say $1, $2) to provide more context to the message. E.g. Format related errors may want to indicate the actual field for which the format is invalid. Client's should use the values in the param array to replace those fields. + items: + type: string + required: + - code + - message + + ErrorRes: + type: object + description: All APIs will return ErrorRes in case of failure which will carry ResponseInfo as metadata and Error object as actual representation of error. In case of bulk apis, some apis may chose to return the array of Error objects to indicate individual failure. + properties: + statusInfo: + $ref: '#/definitions/StatusInfo' + errors: + description: Error response array corresponding to Request Object array. In case of single object submission or _search related paths this may be an array of one error element + type: array + minLength: 1 + items: + $ref: '#/definitions/Error' + required: + - statusInfo \ No newline at end of file diff --git a/build/build-config.yml b/build/build-config.yml index 5b2882dad..3e7071d16 100644 --- a/build/build-config.yml +++ b/build/build-config.yml @@ -17,7 +17,7 @@ # - # - config: - - name: "business-services/billing-service" + - name: "builds/business-services/billing-service" build: - work-dir: "billing-service" image-name: "billing-service" @@ -25,7 +25,7 @@ config: - work-dir: "billing-service/src/main/resources/db" image-name: "billing-service-db" - - name: "business-services/collection-services" + - name: "builds/business-services/collection-services" build: - work-dir: "collection-services" image-name: "collection-services" @@ -33,7 +33,7 @@ config: - work-dir: "collection-services/src/main/resources/db" image-name: "collection-services-db" - - name: "business-services/egf-instrument" + - name: "builds/business-services/egf-instrument" build: - work-dir: "egf-instrument" image-name: "egf-instrument" @@ -41,7 +41,7 @@ config: - work-dir: "egf-instrument/src/main/resources/db" image-name: "egf-instrument-db" - - name: "business-services/egf-master" + - name: "builds/business-services/egf-master" build: - work-dir: "egf-master" image-name: "egf-master" @@ -49,7 +49,7 @@ config: - work-dir: "egf-master/src/main/resources/db" image-name: "egf-master-db" - - name: "business-services/egov-apportion-service" + - name: "builds/business-services/egov-apportion-service" build: - work-dir: "egov-apportion-service" image-name: "egov-apportion-service" @@ -57,7 +57,7 @@ config: - work-dir: "egov-apportion-service/src/main/resources/db" image-name: "egov-apportion-service-db" - - name: "business-services/egov-hrms" + - name: "builds/business-services/egov-hrms" build: - work-dir: "egov-hrms" image-name: "egov-hrms" @@ -65,7 +65,7 @@ config: - work-dir: "egov-hrms/src/main/resources/db" image-name: "egov-hrms-db" - - name: "business-services/finance-collections-voucher-consumer" + - name: "builds/business-services/finance-collections-voucher-consumer" build: - work-dir: "finance-collections-voucher-consumer" image-name: "finance-collections-voucher-consumer" @@ -73,13 +73,13 @@ config: - work-dir: "finance-collections-voucher-consumer/src/main/resources/db" image-name: "finance-collections-voucher-consumer-db" - - name: "business-services/dashboard-analytics" + - name: "builds/business-services/dashboard-analytics" build: - work-dir: "dashboard-analytics" image-name: "dashboard-analytics" dockerfile: "build/maven/Dockerfile" - - name: "business-services/dashboard-ingest" + - name: "builds/business-services/dashboard-ingest" build: - work-dir: "dashboard-ingest" image-name: "dashboard-ingest" diff --git a/dashboard-analytics/pom.xml b/dashboard-analytics/pom.xml new file mode 100644 index 000000000..5e83d1569 --- /dev/null +++ b/dashboard-analytics/pom.xml @@ -0,0 +1,155 @@ + + 4.0.0 + com.tarento + analytics + 0.0.1-SNAPSHOT + + Analytic Service + Project for handling the data for analytics visualization + http://maven.apache.org + + org.springframework.boot + spring-boot-starter-parent + 1.5.3.RELEASE + + + + UTF-8 + UTF-8 + 1.8 + 3.0.1 + + + + + junit + junit + 3.8.1 + test + + + org.springframework.kafka + spring-kafka + 1.1.2.RELEASE + + + org.springframework.boot + spring-boot-starter-web + + + + org.springframework.boot + spring-boot-starter-jdbc + 2.0.0.RELEASE + + + + com.google.code.gson + gson + 2.8.0 + + + + org.springframework.boot + spring-boot-starter-jersey + 1.2.0.RELEASE + + + + com.fasterxml.jackson.dataformat + jackson-dataformat-xml + + + org.projectlombok + lombok + + + org.apache.poi + poi-ooxml + 3.10-FINAL + + + + javax.ws.rs + jsr311-api + 1.1.1 + + + org.apache.cxf + cxf-rt-frontend-jaxws + ${cxf.version} + + + org.apache.cxf + cxf-rt-transports-http + ${cxf.version} + + + org.apache.cxf + cxf-rt-frontend-jaxrs + ${cxf.version} + + + org.apache.cxf + cxf-tools-java2ws + ${cxf.version} + + + org.apache.commons + commons-lang3 + 3.4 + + + + + org.elasticsearch.client + elasticsearch-rest-high-level-client + 6.2.4 + + + org.elasticsearch + elasticsearch + 6.2.4 + + + com.github.fge + json-schema-validator + 2.2.6 + + + + commons-io + commons-io + 2.6 + + + + org.json + json + 20180813 + + + com.googlecode.json-simple + json-simple + 1.1 + + + com.amazonaws + aws-java-sdk + 1.11.550 + + + + + + + org.springframework.boot + spring-boot-maven-plugin + + + + diff --git a/dashboard-analytics/src/main/java/com/tarento/analytics/AnalyticApp.java b/dashboard-analytics/src/main/java/com/tarento/analytics/AnalyticApp.java new file mode 100644 index 000000000..883def0f5 --- /dev/null +++ b/dashboard-analytics/src/main/java/com/tarento/analytics/AnalyticApp.java @@ -0,0 +1,35 @@ +package com.tarento.analytics; + +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.context.annotation.Bean; +import org.springframework.web.client.RestTemplate; +import org.springframework.web.servlet.config.annotation.CorsRegistry; +import org.springframework.web.servlet.config.annotation.WebMvcConfigurer; +import org.springframework.web.servlet.config.annotation.WebMvcConfigurerAdapter; + +import com.tarento.analytics.constant.Constants; + + +@SpringBootApplication +public class AnalyticApp { + public static void main( String[] args ) { + SpringApplication.run(AnalyticApp.class, args); + } + + @Bean + public RestTemplate restTemplate() { + return new RestTemplate(); + } + + @Bean + public WebMvcConfigurer corsConfigurer() { + return new WebMvcConfigurerAdapter() { + @Override + public void addCorsMappings(CorsRegistry registry) { + registry.addMapping("/**").allowedMethods(Constants.GET, Constants.POST,Constants.PUT, Constants.DELETE, Constants.OPTIONS).allowedOrigins("*") + .allowedHeaders("*"); + } + }; + } +} diff --git a/dashboard-analytics/src/main/java/com/tarento/analytics/ConfigurationLoader.java b/dashboard-analytics/src/main/java/com/tarento/analytics/ConfigurationLoader.java new file mode 100644 index 000000000..6353ff4a7 --- /dev/null +++ b/dashboard-analytics/src/main/java/com/tarento/analytics/ConfigurationLoader.java @@ -0,0 +1,97 @@ +package com.tarento.analytics; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.apache.commons.io.IOUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.core.io.Resource; +import org.springframework.core.io.ResourceLoader; +import org.springframework.core.io.support.ResourcePatternUtils; +import org.springframework.stereotype.Component; + +import javax.annotation.PostConstruct; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.Charset; +import java.util.HashMap; +import java.util.Map; + +@Component("configurationLoader") +public class ConfigurationLoader { + + private static Logger logger = LoggerFactory.getLogger(ConfigurationLoader.class); + private Map nameContentMap = new HashMap<>(); + @Autowired + private ResourceLoader resourceLoader; + @Autowired + private ObjectMapper objectMapper; + + @Value("${config.schema.paths}") + private String RESOURCE_LOCATION; + + // private static final String RESOURCE_LOCATION = "file://home/darshan/Tarento/DataPlatform/analytics2/analytics/src/main/resources/schema/*.json"; + public static final String ROLE_DASHBOARD_CONFIG = "RoleDashboardMappingsConf.json"; + public static final String MASTER_DASHBOARD_CONFIG = "MasterDashboardConfig.json"; + + + /** + * Loads config resources + * @throws Exception + */ + @PostConstruct + public void loadResources() throws Exception { + Resource[] resources = getResources(RESOURCE_LOCATION); + + for (Resource resource : resources) { + String jsonContent = getContent(resource); + ObjectNode jsonNode = (ObjectNode) objectMapper.readTree(jsonContent); + nameContentMap.put(resource.getFilename(), jsonNode); + } + logger.info("Number of resources loaded " + nameContentMap.size()); + + } + + /** + * Obtains a ObjectNode w.r.t given resource/file name in classpath*:schema + * @param name + * @return + */ + public ObjectNode get(String name) { + return nameContentMap.get(name); + } + + /** + * Loads all the resources/files with a given pattern *.json + * @param pattern path with *json + * @return + * @throws IOException + */ + private Resource[] getResources(String pattern) throws IOException { + Resource[] resources = ResourcePatternUtils.getResourcePatternResolver(resourceLoader).getResources(pattern); + return resources; + } + + /** + * Returns a content of resource + * + * @param resource + * @return + */ + private String getContent(Resource resource) { + String content = null; + try { + InputStream is = resource.getInputStream(); + byte[] encoded = IOUtils.toByteArray(is); + content = new String(encoded, Charset.forName("UTF-8")); + + } catch (IOException e) { + logger.error("Cannot load resource " + resource.getFilename()); + + } + return content; + } + +} diff --git a/dashboard-analytics/src/main/java/com/tarento/analytics/constant/Constants.java b/dashboard-analytics/src/main/java/com/tarento/analytics/constant/Constants.java new file mode 100644 index 000000000..55b9a7a70 --- /dev/null +++ b/dashboard-analytics/src/main/java/com/tarento/analytics/constant/Constants.java @@ -0,0 +1,134 @@ +package com.tarento.analytics.constant; + +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +public class Constants { + /** + * Allowed Origins for CORS Bean + */ + public static final String GET = "GET"; + public static final String POST = "POST"; + public static final String PUT = "PUT"; + public static final String DELETE = "DELETE"; + public static final String OPTIONS = "OPTIONS"; + + + public static int UNAUTHORIZED_ID = 401; + public static int SUCCESS_ID = 200; + public static int FAILURE_ID = 320; + public static String UNAUTHORIZED = "Invalid credentials. Please try again."; + public static String PROCESS_FAIL = "Process failed, Please try again."; + public static String SUCCESS= "success"; + + + //chart format + + public static final String D3 = "d3"; + public static final String CHARTJS = "chartjs"; + + //chart type + public static final String BAR = "bar"; + public static final String PIE ="pie"; + public static final String STACKEDBAR ="stackedbar"; + public static final String LINE ="line"; + public static final String HORIZONTAL_BAR="horizontalBar"; + public static final String DOUGHNUT="doughnut"; + public static final String Heat = "heat"; + public static final String RADAR ="radar"; + + public static final Long FEEDBACK_MESSAGE_TIMEOUT = 2000l; + + public static final String STORE_ID = "storeId"; + + public static final String PLACEMENTS_DASHBOARD = "DASHBOARD"; + public static final String PLACEMENTS_HOME = "HOME"; + + public static final List RATING_LIST = new ArrayList<>(Arrays.asList(1l,2l,3l,4l,5l)); + public static final List RATING_LIST_STRING = new ArrayList<>(Arrays.asList("1","2","3","4","5")); + public static final List RATING_LIST_STRING_STAR = new ArrayList<>(Arrays.asList("1 Star","2 Star","3 Star","4 Star","5 Star")); + + public interface Modules { + final static String HOME_REVENUE = "HOME_REVENUE"; + final static String HOME_SERVICES = "HOME_SERVICES"; + final static String COMMON = "COMMON"; + final static String PT = "PT"; + final static String TL = "TL"; + } + + public interface KafkaTopics { + final static String NEW_CONTENT_MESSAGE = "SaveContent"; + final static String SIMULATOR_TRANSACTION = "SaveTransaction"; + } + + public interface ConfigurationFiles { + final static String CHART_API_CONFIG = "ChartApiConfig.json"; + } + + public interface JsonPaths { + final static String CHART_TYPE = "chartType"; + final static String QUERIES = "queries"; + final static String AGGREGATION_QUERY= "aggrQuery"; + final static String INDEX_NAME = "indexName"; + final static String REQUEST_QUERY_MAP = "requestQueryMap"; + final static String DATE_REF_FIELD = "dateRefField"; + final static String AGGS = "aggs"; + final static String AGGREGATIONS = "aggregations" ; + final static String MODULE = "module"; + final static String INTERVAL_VAL = "intervalvalue"; + final static String INTERVAL = "interval"; + final static String IS_MDMS_ENABALED = "isMdmsEnabled"; + final static String INSIGHT = "insight"; + final static String DAYS = "days"; + final static String TRANSLATE_CODE = "translateTenantCode"; + + } + + public interface Filters { + final static String MODULE = "module"; + final static String FILTER_ALL = "*"; + } + + public interface Catagory { + final static String SEVICE = "service"; + final static String REVENUE = "revenue"; + } + + public interface DashBoardConfig { + final static String ROLES = "roles"; + final static String ROLE_ID = "roleId"; + final static String DASHBOARDS = "dashboards"; + final static String VISUALISATIONS = "visualizations"; + final static String NAME = "name"; + final static String ID = "id"; + final static String TITLE = "title"; + + } + public interface MDMSKeys { + final static String CODE = "code"; + final static String DISTRICT_CODE = "districtCode"; + final static String DDR_NAME = "ddrName"; + final static String TENANTS = "tenants"; + final static String KEY = "key"; + final static String TENANT_ID = "tenantId"; + + } + + public enum Interval { + week, month, year, day + } + public interface PostAggregationTheories { + final static String RESPONSE_DIFF_DATES = "repsonseToDifferenceOfDates"; + } + + public enum ClienServiceType { + DEFAULT_CLIENT , + MDMS_CLIENT + } + + public static final SimpleDateFormat DASHBOARD_DATE_FORMAT = new SimpleDateFormat("MMM dd, yyyy"); + + +} diff --git a/dashboard-analytics/src/main/java/com/tarento/analytics/constant/ElasticSearchConstants.java b/dashboard-analytics/src/main/java/com/tarento/analytics/constant/ElasticSearchConstants.java new file mode 100644 index 000000000..09963b4dc --- /dev/null +++ b/dashboard-analytics/src/main/java/com/tarento/analytics/constant/ElasticSearchConstants.java @@ -0,0 +1,22 @@ +package com.tarento.analytics.constant; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +public class ElasticSearchConstants { + public static final String LTE = "<="; + public static final String LT = "<"; + public static final String GTE = ">="; + public static final String GT = ">"; + public static final String ASC_ORDER = "ASC"; + public static final String STARTS_WITH = "startsWith"; + public static final String ENDS_WITH = "endsWith"; + public static final List upsertResults = + new ArrayList<>(Arrays.asList("CREATED", "UPDATED", "NOOP")); + public static final String SOFT_MODE = "soft"; + public static final String RAW_APPEND = ".raw"; + public static final String DAY_OF_WEEK = "dayOfWeek"; + public static final String DAY = "day"; + public static final String HOUR = "hour"; +} diff --git a/dashboard-analytics/src/main/java/com/tarento/analytics/constant/ErrorCode.java b/dashboard-analytics/src/main/java/com/tarento/analytics/constant/ErrorCode.java new file mode 100644 index 000000000..56a8e5cc4 --- /dev/null +++ b/dashboard-analytics/src/main/java/com/tarento/analytics/constant/ErrorCode.java @@ -0,0 +1,10 @@ +package com.tarento.analytics.constant; + +public class ErrorCode { + + public static final String ERR320= "320"; + + + + +} diff --git a/dashboard-analytics/src/main/java/com/tarento/analytics/controllers/DashboardController.java b/dashboard-analytics/src/main/java/com/tarento/analytics/controllers/DashboardController.java new file mode 100644 index 000000000..079e3ae1d --- /dev/null +++ b/dashboard-analytics/src/main/java/com/tarento/analytics/controllers/DashboardController.java @@ -0,0 +1,221 @@ +package com.tarento.analytics.controllers; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import com.tarento.analytics.org.service.ClientServiceFactory; +import com.tarento.analytics.service.AmazonS3ClientService; +import org.apache.commons.lang3.StringUtils; +import org.json.JSONArray; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.http.HttpStatus; +import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.*; +import org.springframework.web.context.request.ServletWebRequest; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.tarento.analytics.constant.Constants; +import com.tarento.analytics.constant.ErrorCode; +import com.tarento.analytics.dto.AggregateRequestDto; +import com.tarento.analytics.dto.AggregateRequestDtoV3; +import com.tarento.analytics.dto.RequestDto; +import com.tarento.analytics.dto.RequestDtoV3; +import com.tarento.analytics.dto.RoleDto; +import com.tarento.analytics.dto.UserDto; +import com.tarento.analytics.exception.AINException; +import com.tarento.analytics.org.service.ClientService; +import com.tarento.analytics.service.MetadataService; +import com.tarento.analytics.utils.PathRoutes; +import com.tarento.analytics.utils.ResponseGenerator; +import org.springframework.web.multipart.MultipartFile; + +@RestController +@RequestMapping(PathRoutes.DashboardApi.DASHBOARD_ROOT_PATH) +public class DashboardController { + + public static final Logger logger = LoggerFactory.getLogger(DashboardController.class); + + @Autowired + private MetadataService metadataService; + @Autowired + private AmazonS3ClientService amazonS3ClientService; + +/* @Autowired + private ClientService clientService;*/ + + @Autowired + private ClientServiceFactory clientServiceFactory; + + @RequestMapping(value = PathRoutes.DashboardApi.FILE_PATH, method = RequestMethod.POST) + public Map uploadFile(@RequestPart(value = "file") MultipartFile file) + { + Map response = new HashMap<>(); + try{ + String imgUrl = this.amazonS3ClientService.uploadFileToS3Bucket(file, true); + response.put("message", "file [" + file.getOriginalFilename() + "] uploading request submitted successfully."); + response.put("url", imgUrl); + }catch (Exception e){ + logger.error("S3 file upload : "+e.getMessage()); + response.put("message", e.getMessage()); + response.put("url", ""); + } + + return response; + } + + @DeleteMapping(value = PathRoutes.DashboardApi.FILE_PATH) + public Map deleteFile(@RequestParam("file_name") String fileName) + { + Map response = new HashMap<>(); + try{ + this.amazonS3ClientService.deleteFileFromS3Bucket(fileName); + response.put("message", "file [" + fileName + "] removing request submitted successfully."); + }catch (Exception e ){ + logger.error("S3 file upload : "+e.getMessage()); + response.put("message", e.getMessage()); + + } + return response; + + } + + @GetMapping(value = PathRoutes.DashboardApi.TEST_PATH, produces = MediaType.APPLICATION_JSON_VALUE) + public String getTest() throws JsonProcessingException { + return ResponseGenerator.successResponse("success"); + + } + + @RequestMapping(value = PathRoutes.DashboardApi.GET_DASHBOARD_CONFIG + "/{dashboardId}", method = RequestMethod.GET) + public String getDashboardConfiguration(@PathVariable String dashboardId, @RequestParam(value="catagory", required = false) String catagory, @RequestHeader(value = "x-user-info", required = false) String xUserInfo) + throws AINException, IOException { + Gson gson = new GsonBuilder().setPrettyPrinting().create(); + UserDto user = new UserDto(); + user.setId(new Long("10007")); + user.setOrgId("1"); + user.setCountryCode(""); + RoleDto role = new RoleDto(); + role.setId(new Long("6")); + role.setName("HR User"); + List roles = new ArrayList<>(); + roles.add(role); + user.setRoles(roles); + //gson.fromJson(xUserInfo, UserDto.class); + + return ResponseGenerator.successResponse(metadataService.getDashboardConfiguration(dashboardId, catagory, user.getRoles())); + } + + @RequestMapping(value = PathRoutes.DashboardApi.GET_CHART_V2, method = RequestMethod.POST) + public String getVisualizationChartV2( @RequestBody RequestDto requestDto, @RequestHeader(value = "x-user-info", required = false) String xUserInfo, ServletWebRequest request) + throws IOException { + + /*logger.info("Request Detail:" + requestDto); + Gson gson = new GsonBuilder().setPrettyPrinting().create(); + UserDto user = gson.fromJson(xUserInfo, UserDto.class);*/ + + UserDto user = new UserDto(); + logger.info("user"+xUserInfo); + + //Getting the request information only from the Full Request + AggregateRequestDto requestInfo = requestDto.getAggregationRequestDto(); + Map headers = requestDto.getHeaders(); + //requestInfo.getFilters().putAll(headers); + String response = ""; + try { + if (headers.isEmpty()) { + logger.error("Please provide header details"); + throw new AINException(ErrorCode.ERR320, "header is missing"); + } + if (headers.get("tenantId") == null) { + logger.error("Please provide tenant ID details"); + throw new AINException(ErrorCode.ERR320, "tenant is missing"); + + } + + if(requestDto.getAggregationRequestDto() == null) { + logger.error("Please provide requested Visualization Details"); + throw new AINException(ErrorCode.ERR320, "Visualization Request is missing"); + } + /*if(requestDto.getAggregationRequestDto().getRequestId() == null) { + logger.error("Please provide Request ID"); + throw new AINException(ErrorCode.ERR320, "Request ID is missing. Insights will not work"); + }*/ + + + // To be removed once the development is complete + if(StringUtils.isBlank(requestInfo.getModuleLevel())) { + requestInfo.setModuleLevel(Constants.Modules.HOME_REVENUE); + } + + Object responseData = clientServiceFactory.get(requestInfo.getVisualizationCode()).getAggregatedData(requestInfo, user.getRoles()); + response = ResponseGenerator.successResponse(responseData); + + } catch (AINException e) { + logger.error("error while executing api getVisualizationChart"); + response = ResponseGenerator.failureResponse(e.getErrorCode(), e.getErrorMessage()); + } + return response; + } + +/* + @RequestMapping(value = PathRoutes.DashboardApi.GET_CHART_V3, method = RequestMethod.POST) + public String getVisualizationChartV3(@RequestBody RequestDtoV3 requestDtoV3, @RequestHeader(value = "x-user-info", required = false) String xUserInfo, ServletWebRequest request) + throws IOException { + + */ +/*logger.info("Request Detail:" + requestDto); + Gson gson = new GsonBuilder().setPrettyPrinting().create(); + UserDto user = gson.fromJson(xUserInfo, UserDto.class);*//* + + + UserDto user = new UserDto(); + logger.info("user"+xUserInfo); + + //Getting the request information only from the Full Request + AggregateRequestDtoV3 requestInfoV3 = requestDtoV3.getAggregationRequestDto(); + Map headers = requestDtoV3.getHeaders(); + //requestInfo.getFilters().putAll(headers); + String response = ""; + try { + if (headers.isEmpty()) { + logger.error("Please provide header details"); + throw new AINException(ErrorCode.ERR320, "header is missing"); + } + if (headers.get("tenantId") == null) { + logger.error("Please provide tenant ID details"); + throw new AINException(ErrorCode.ERR320, "tenant is missing"); + } + // To be removed once the development is complete + if(StringUtils.isBlank(requestInfoV3.getModuleLevel())) { + requestInfoV3.setModuleLevel(Constants.Modules.HOME_REVENUE); + } + + List responseDataList = new ArrayList<>(); + if(requestInfoV3 !=null && requestInfoV3.getVisualizations() != null && requestInfoV3.getVisualizations().size() > 0) { + for (int i = 0; i < requestInfoV3.getVisualizations().size(); i++) { + AggregateRequestDto requestInfo = new AggregateRequestDto(requestInfoV3, + requestInfoV3.getVisualizations().get(i).getType(), requestInfoV3.getVisualizations().get(i).getCode()); + Object responseData = clientService.getAggregatedData(requestInfo, user.getRoles()); + responseDataList.add(responseData); + } + + } + response = ResponseGenerator.successResponse(responseDataList); + } catch (AINException e) { + logger.error("error while executing api getVisualizationChart"); + response = ResponseGenerator.failureResponse(e.getErrorCode(), e.getErrorMessage()); + } + return response; + } +*/ + + + +} diff --git a/dashboard-analytics/src/main/java/com/tarento/analytics/dao/ElasticSearchDao.java b/dashboard-analytics/src/main/java/com/tarento/analytics/dao/ElasticSearchDao.java new file mode 100644 index 000000000..1d72b6357 --- /dev/null +++ b/dashboard-analytics/src/main/java/com/tarento/analytics/dao/ElasticSearchDao.java @@ -0,0 +1,41 @@ +package com.tarento.analytics.dao; + +import java.util.List; +import java.util.Map; + +import org.elasticsearch.action.search.MultiSearchResponse; +import org.elasticsearch.action.search.SearchRequest; + +import com.tarento.analytics.dto.AggregateRequestDto; +import com.tarento.analytics.dto.AggregateRequestDto; +import com.tarento.analytics.dto.CummulativeDataRequestDto; +import com.tarento.analytics.dto.SearchDto; +import com.tarento.analytics.model.ElasticSearchDictator; + + +public interface ElasticSearchDao { + + public Map getDataByIdentifier(String index, String type, String identifier); + + public Map searchData(String index, String type, Map searchData); + + public Map complexSearch(SearchDto searchDTO, String index, String... type); + + public boolean healthCheck(); + + /*public ElasticSearchDictator createSearchDictator(AggregateRequestDto dto, String tenant) throws Exception ; + + public ElasticSearchDictator createSearchDictator(AggregateRequestDto dto, String indexName, String documentType, String filterDateField) throws Exception ; + */ + public ElasticSearchDictator createSearchDictatorV2(AggregateRequestDto dto, String indexName, String documentType, String filterDateField) throws Exception ; + + public SearchRequest buildElasticSearchQuery(ElasticSearchDictator dictator); + + public MultiSearchResponse executeMultiSearchRequest(List searchRequestList, String tenant); + + //public ElasticSearchDictator createSearchDictator(String indexName, String documentType, CummulativeDataRequestDto dto, String tenant) throws Exception ; + + MultiSearchResponse executeMultiSearchRequest(List searchRequestList, Boolean primaryOrNot); + +} + diff --git a/dashboard-analytics/src/main/java/com/tarento/analytics/dao/impl/ElasticSearchDaoImpl.java b/dashboard-analytics/src/main/java/com/tarento/analytics/dao/impl/ElasticSearchDaoImpl.java new file mode 100644 index 000000000..732d2c1a1 --- /dev/null +++ b/dashboard-analytics/src/main/java/com/tarento/analytics/dao/impl/ElasticSearchDaoImpl.java @@ -0,0 +1,1287 @@ +package com.tarento.analytics.dao.impl; + +import java.io.IOException; +import java.net.MalformedURLException; +import java.net.URL; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.stream.Collectors; + +import org.apache.commons.lang3.StringUtils; +import org.apache.http.HttpHost; +import org.elasticsearch.action.get.GetRequest; +import org.elasticsearch.action.get.GetResponse; +import org.elasticsearch.action.search.MultiSearchRequest; +import org.elasticsearch.action.search.MultiSearchResponse; +import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.client.RestClient; +import org.elasticsearch.client.RestHighLevelClient; +import org.elasticsearch.index.query.BoolQueryBuilder; +import org.elasticsearch.index.query.ExistsQueryBuilder; +import org.elasticsearch.index.query.MatchQueryBuilder; +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.index.query.RangeQueryBuilder; +import org.elasticsearch.index.query.TermQueryBuilder; +import org.elasticsearch.index.query.TermsQueryBuilder; +import org.elasticsearch.search.SearchHit; +import org.elasticsearch.search.SearchHits; +import org.elasticsearch.search.aggregations.AggregationBuilder; +import org.elasticsearch.search.aggregations.AggregationBuilders; +import org.elasticsearch.search.aggregations.BucketOrder; +import org.elasticsearch.search.aggregations.PipelineAggregationBuilder; +import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramAggregationBuilder; +import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval; +import org.elasticsearch.search.aggregations.bucket.histogram.ExtendedBounds; +import org.elasticsearch.search.aggregations.bucket.nested.NestedAggregationBuilder; +import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder; +import org.elasticsearch.search.aggregations.metrics.avg.AvgAggregationBuilder; +import org.elasticsearch.search.aggregations.metrics.sum.SumAggregationBuilder; +import org.elasticsearch.search.aggregations.metrics.valuecount.ValueCountAggregationBuilder; +import org.elasticsearch.search.aggregations.pipeline.cumulativesum.CumulativeSumPipelineAggregationBuilder; +import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.elasticsearch.search.sort.SortBuilder; +import org.elasticsearch.search.sort.SortBuilders; +import org.elasticsearch.search.sort.SortOrder; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.tarento.analytics.constant.ElasticSearchConstants; +import com.tarento.analytics.dao.ElasticSearchDao; +import com.tarento.analytics.dto.AggregateRequestDto; +import com.tarento.analytics.dto.SearchDto; +import com.tarento.analytics.model.ElasticSearchDictator; +import com.tarento.analytics.utils.ElasticProperties; +import com.tarento.analytics.utils.ElasticSearchClient; + +@Component +public class ElasticSearchDaoImpl implements ElasticSearchDao { + + @Autowired + private ElasticSearchClient elasticSearchClient; + + @Autowired + public ObjectMapper mapper; + + private static String elasticHost; + private static int elasticPort; + private static String alternateHost; + private static int alternatePort; + private static final String REST_SCHEME = "http"; + private static final String REST_SCHEME2 = "https"; + private String transactionElasticIndexName; + private static String transactionElasticDocType; + private String ratingElasticIndexName; + private static String ratingElasticDocType; + private static RestHighLevelClient client; + private static RestHighLevelClient alternateClient; + private String indexName; + private String docType; + + public String getIndexName() { + return indexName; + } + + public void setIndexName(String indexName) { + this.indexName = indexName; + } + + public String getDocType() { + return docType; + } + + public void setDocType(String docType) { + this.docType = docType; + } + + public static final Logger logger = LoggerFactory.getLogger(ElasticSearchDaoImpl.class); + + public ElasticSearchDaoImpl(@Value("${services.esindexer.host.name}") String elasticHost, + @Value("${services.esindexer.host.port}") int elasticPort) throws MalformedURLException { + this.elasticHost = elasticHost; + this.elasticPort = elasticPort; + this.client = getClientForElastic(); + + } + + @Override + public Map getDataByIdentifier(String index, String type, String identifier) { + RestHighLevelClient client = elasticSearchClient.getClient(); + long startTime = System.currentTimeMillis(); + logger.info("ElasticSearchUtil getDataByIdentifier method started at ==" + startTime + " for Type " + type); + GetResponse response = null; + + try { + if (StringUtils.isBlank(index) || StringUtils.isBlank(identifier)) { + logger.error("Invalid request is coming."); + return new HashMap<>(); + } else if (StringUtils.isBlank(type)) { + response = client.get(new GetRequest(index).type(type)); + + } else { + response = client.get(new GetRequest(index, type, identifier)); + } + if (response == null || null == response.getSource()) { + return new HashMap<>(); + } + long stopTime = System.currentTimeMillis(); + long elapsedTime = stopTime - startTime; + logger.info("ElasticSearchUtil getDataByIdentifier method end at ==" + stopTime + " for Type " + type + + " ,Total time elapsed = " + elapsedTime); + return response.getSource(); + } catch (IOException ex) { + + } finally { + try { + client.close(); + } catch (IOException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + } + return null; + } + + @Override + public Map searchData(String index, String type, Map searchData) { + RestHighLevelClient client = elasticSearchClient.getClient(); + + long startTime = System.currentTimeMillis(); + logger.info("ElasticSearchUtil searchData method started at ==" + startTime + " for Type " + type); + SearchSourceBuilder sourceBuilder = new SearchSourceBuilder(); + Iterator> itr = searchData.entrySet().iterator(); + while (itr.hasNext()) { + Entry entry = itr.next(); + sourceBuilder.query(QueryBuilders.commonTermsQuery(entry.getKey(), entry.getValue())); + } + + SearchResponse sr = null; + try { + sr = client.search(new SearchRequest(index).types(type).source(sourceBuilder)); + + } catch (IOException ex) { + logger.error("Error while execution in Elasticsearch", ex); + } + + if (sr.getHits() == null || sr.getHits().getTotalHits() == 0) { + return new HashMap<>(); + } + sr.getHits().getAt(0).getSourceAsMap(); + long stopTime = System.currentTimeMillis(); + long elapsedTime = stopTime - startTime; + + logger.info("ElasticSearchUtil searchData method end at ==" + stopTime + " for Type " + type + + " ,Total time elapsed = " + elapsedTime); + try { + client.close(); + } catch (IOException e) { + // TODO Auto-generated catch block + logger.error("Error while closing the client"); + } + return sr.getAggregations().asList().get(0).getMetaData(); + } + + @SuppressWarnings("rawtypes") + @Override + public Map complexSearch(SearchDto searchDTO, String index, String... type) { + RestHighLevelClient client = elasticSearchClient.getClient(); + + long startTime = System.currentTimeMillis(); + logger.info("ElasticSearchUtil complexSearch method started at ==" + startTime); + + // Map constraintsMap = getConstraints(searchDTO); + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder().size(0); + BoolQueryBuilder query = new BoolQueryBuilder(); + + if (!StringUtils.isBlank(searchDTO.getQuery())) { + query.must(QueryBuilders.simpleQueryStringQuery(searchDTO.getQuery()).field("all_fields")); + } + + // apply sorting + if (searchDTO.getSortBy() != null && searchDTO.getSortBy().size() > 0) { + + for (Map.Entry entry : searchDTO.getSortBy().entrySet()) { + SortBuilder sortB = SortBuilders.fieldSort(entry.getKey()).order(getSortOrder(entry.getValue())); + searchSourceBuilder.sort(sortB); + + } + } + + // apply the fields filter + searchSourceBuilder.fetchSource( + searchDTO.getFields() != null ? searchDTO.getFields().stream().toArray(String[]::new) : null, + searchDTO.getExcludedFields() != null ? searchDTO.getExcludedFields().stream().toArray(String[]::new) + : null); + + // setting the offset + if (searchDTO.getOffset() != null) { + searchSourceBuilder.from(searchDTO.getOffset()); + } + + // setting the limit + if (searchDTO.getLimit() != null) { + searchSourceBuilder.size(searchDTO.getLimit()); + } + + // apply additional properties + if (searchDTO.getAdditionalProperties() != null && searchDTO.getAdditionalProperties().size() > 0) { + for (Map.Entry entry : searchDTO.getAdditionalProperties().entrySet()) { + addAdditionalProperties(query, entry); + } + } + + searchSourceBuilder.query(query); + List finalFacetList = new ArrayList(); + + if (null != searchDTO.getFacets() && !searchDTO.getFacets().isEmpty()) { + // addAggregations(searchSourceBuilder, searchDTO.getFacets()); + } + logger.info("calling search builder======" + searchSourceBuilder.toString()); + SearchResponse response = null; + SearchRequest searchReq = new SearchRequest(index).types(type).source(searchSourceBuilder); + List> esSource = new ArrayList<>(); + + try { + response = client.search(searchReq); + long count = 0; + + if (response != null) { + SearchHits hits = response.getHits(); + count = hits.getTotalHits(); + for (SearchHit hit : hits) { + esSource.add(hit.getSourceAsMap()); + } + } + } catch (IOException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + return null; + + } + + @SuppressWarnings("unchecked") + private void addAggregations(SearchSourceBuilder searchSourceBuilder, Map aggregations) { + long startTime = System.currentTimeMillis(); + logger.info("ElasticSearchUtil addAggregations method started at ==" + startTime); + for (Map.Entry entry : aggregations.entrySet()) { + + String key = entry.getKey(); + Map aggregationInfo = (Map) entry.getValue(); + for (Map.Entry en : aggregations.entrySet()) { + if ("DATE_HISTOGRAM".equalsIgnoreCase(en.getKey())) { + Map aggsVal = (Map) en.getValue(); + DateHistogramInterval dateHistogramInterval = null; + if (aggsVal.get("interval").equals("day")) { + dateHistogramInterval = DateHistogramInterval.DAY; + } else if (aggsVal.get("interval").equals("hour")) { + dateHistogramInterval = DateHistogramInterval.HOUR; + + } else if (aggsVal.get("interval").equals("month")) { + dateHistogramInterval = DateHistogramInterval.MONTH; + + } else if (aggsVal.get("interval").equals("year")) { + dateHistogramInterval = DateHistogramInterval.YEAR; + + } + searchSourceBuilder.aggregation(AggregationBuilders.dateHistogram(key).field(aggsVal.get("field")) + .dateHistogramInterval(dateHistogramInterval)); + + } else if ("TERMS".equalsIgnoreCase(en.getKey())) { + Map aggVal = (Map) en.getValue(); + for (Map.Entry entryS : aggVal.entrySet()) { + + /* + * searchSourceBuilder.aggregation(AggregationBuilders. + * dateHistogram(key).field(aggsVal.get("field")) + * .dateHistogramInterval(Date)); + */ + + } + + } + } + + } + long stopTime = System.currentTimeMillis(); + long elapsedTime = stopTime - startTime; + logger.info("ElasticSearchUtil addAggregations method end at ==" + stopTime + " ,Total time elapsed = " + + elapsedTime); + } + + @SuppressWarnings("unchecked") + private void addAdditionalProperties(BoolQueryBuilder query, Entry entry) { + long startTime = System.currentTimeMillis(); + logger.info("ElasticSearchUtil addAdditionalProperties method started at ==" + startTime); + String key = entry.getKey(); + + if (key.equalsIgnoreCase("FILTERS")) { + + Map filters = (Map) entry.getValue(); + for (Map.Entry en : filters.entrySet()) { + createFilterESOpperation(en, query); + } + } else if (key.equalsIgnoreCase("EXISTS") || key.equalsIgnoreCase("NOT_EXISTS")) { + createESOpperation(entry, query); + } + long stopTime = System.currentTimeMillis(); + long elapsedTime = stopTime - startTime; + logger.info("ElasticSearchUtil addAdditionalProperties method end at ==" + stopTime + " ,Total time elapsed = " + + elapsedTime); + } + + /** Method to create EXISTS and NOT EXIST FILTER QUERY . */ + @SuppressWarnings("unchecked") + private void createESOpperation(Entry entry, BoolQueryBuilder query) { + + String operation = entry.getKey(); + List existsList = (List) entry.getValue(); + + if (operation.equalsIgnoreCase("EXISTS")) { + for (String name : existsList) { + query.must(createExistQuery(name)); + } + } else if (operation.equalsIgnoreCase("NOT_EXISTS")) { + for (String name : existsList) { + query.mustNot(createExistQuery(name)); + } + } + } + + /** Method to create CommonTermQuery , multimatch and Range Query. */ + @SuppressWarnings("unchecked") + private void createFilterESOpperation(Entry entry, BoolQueryBuilder query) { + + String key = entry.getKey(); + Object val = entry.getValue(); + if (val instanceof List) { + if (!((List) val).isEmpty()) { + if (((List) val).get(0) instanceof String) { + ((List) val).replaceAll(String::toLowerCase); + query.must(createTermsQuery(key, (List) val)); + } else { + query.must(createTermsQuery(key, (List) val)); + } + } + } else if (val instanceof Map) { + Map value = (Map) val; + Map rangeOperation = new HashMap<>(); + Map lexicalOperation = new HashMap<>(); + for (Map.Entry it : value.entrySet()) { + String operation = it.getKey(); + if (operation.startsWith(ElasticSearchConstants.LT) + || operation.startsWith(ElasticSearchConstants.GT)) { + rangeOperation.put(operation, it.getValue()); + } else if (operation.startsWith(ElasticSearchConstants.STARTS_WITH) + || operation.startsWith(ElasticSearchConstants.ENDS_WITH)) { + lexicalOperation.put(operation, it.getValue()); + } + } + if (!(rangeOperation.isEmpty())) { + query.must(createRangeQuery(key, rangeOperation)); + } + if (!(lexicalOperation.isEmpty())) { + query.must(createLexicalQuery(key, lexicalOperation)); + } + + } else if (val instanceof String) { + query.must(createTermQuery(key, ((String) val).toLowerCase())); + } else { + query.must(createTermQuery(key, val)); + } + } + + private static TermQueryBuilder createTermQuery(String name, Object text) { + + return QueryBuilders.termQuery(name, text); + + } + + @SuppressWarnings("unchecked") + private static TermsQueryBuilder createTermsQuery(String key, List values) { + + return QueryBuilders.termsQuery(key, (values).stream().toArray(Object[]::new)); + + } + + private static Map getConstraints(SearchDto searchDTO) { + if (null != searchDTO.getSoftConstraints() && !searchDTO.getSoftConstraints().isEmpty()) { + return searchDTO.getSoftConstraints().entrySet().stream() + .collect(Collectors.toMap(e -> e.getKey(), e -> e.getValue().floatValue())); + } + return Collections.emptyMap(); + } + + @Override + public boolean healthCheck() { + // TODO Auto-generated method stub + return false; + } + + /// New Methods for Elastic Search Query Builders + + private RangeQueryBuilder createRangeQuery(String name, Map rangeOperation) { + + RangeQueryBuilder rangeQueryBuilder = QueryBuilders.rangeQuery(name); + for (Map.Entry it : rangeOperation.entrySet()) { + if (it.getKey().equalsIgnoreCase(ElasticSearchConstants.LTE)) { + rangeQueryBuilder.lte(it.getValue()); + } else if (it.getKey().equalsIgnoreCase(ElasticSearchConstants.LT)) { + rangeQueryBuilder.lt(it.getValue()); + } else if (it.getKey().equalsIgnoreCase(ElasticSearchConstants.GTE)) { + rangeQueryBuilder.gte(it.getValue()); + } else if (it.getKey().equalsIgnoreCase(ElasticSearchConstants.GT)) { + rangeQueryBuilder.gt(it.getValue()); + } + } + + return rangeQueryBuilder; + } + + private SortOrder getSortOrder(String value) { + return value.equalsIgnoreCase(ElasticSearchConstants.ASC_ORDER) ? SortOrder.ASC : SortOrder.DESC; + } + + private QueryBuilder createLexicalQuery(String key, Map rangeOperation) { + QueryBuilder queryBuilder = null; + for (Map.Entry it : rangeOperation.entrySet()) { + if (it.getKey().equalsIgnoreCase(ElasticSearchConstants.STARTS_WITH)) { + + queryBuilder = QueryBuilders.prefixQuery(key, (String) it.getValue()); + } else if (it.getKey().equalsIgnoreCase(ElasticSearchConstants.ENDS_WITH)) { + String endsWithRegex = "~" + it.getValue(); + queryBuilder = QueryBuilders.regexpQuery(key, endsWithRegex); + } + } + return queryBuilder; + } + + private ExistsQueryBuilder createExistQuery(String name) { + return QueryBuilders.existsQuery(name); + } +/* + + @SuppressWarnings("unchecked") + @Override + public ElasticSearchDictator createSearchDictator(AggregateRequestDto dto, String tenant) throws Exception { + ElasticSearchDictator dictator = new ElasticSearchDictator(); + if (dto.getServiceApi().equals(ServiceApiConstants.CSAT_STORE_WRT_TIME) + || dto.getServiceApi().equals(ServiceApiConstants.CSAT_DAY_WISE) + || dto.getServiceApi().equals(ServiceApiConstants.CSAT_STORE) + || dto.getServiceApi().equals(ServiceApiConstants.CSAT_STORE_REASON) + || dto.getServiceApi().equals(ServiceApiConstants.CSAT_GENDER) + || dto.getServiceApi().equals(ServiceApiConstants.CSAT_AGEGROUP) + || dto.getServiceApi().equals(ServiceApiConstants.CSAT_STORE_TIME) + || dto.getServiceApi().equals(ServiceApiConstants.CSAT_TOP_PERFORMING) + || dto.getServiceApi().equals(ServiceApiConstants.CSAT_LEAST_PERFORMING)) { + dictator.setIndexName(tenant + ratingElasticIndexName); + dictator.setDocumentType(ratingElasticDocType); + } else { + dictator.setIndexName(tenant + transactionElasticIndexName); + dictator.setDocumentType(transactionElasticDocType); + } + if (StringUtils.isNotBlank(dto.getServiceApi())) { + dictator.setVisualisationName(dto.getServiceApi()); + } + Map>> queryMap = new HashMap<>(); + if (dto.getCustomData() != null) { + for (Map.Entry entry : dto.getCustomData().entrySet()) { + if (StringUtils.isNotBlank(entry.getKey()) && entry.getValue() != null) { + List valueList = new ArrayList<>(); + + if (entry.getValue() instanceof ArrayList) { + + List valueArray = (ArrayList) entry.getValue(); + + for (Object value : valueArray) { + valueList.add(value); + } + } else { + valueList.add(entry.getValue()); + } + if (!valueList.isEmpty()) { + String entryKey = entry.getKey(); + if (dto.getServiceApi().equals(ServiceApiConstants.CSAT_STORE_WRT_TIME) + || dto.getServiceApi().equals(ServiceApiConstants.CSAT_DAY_WISE) + || dto.getServiceApi().equals(ServiceApiConstants.CSAT_STORE) + || dto.getServiceApi().equals(ServiceApiConstants.CSAT_STORE_REASON) + || dto.getServiceApi().equals(ServiceApiConstants.CSAT_GENDER) + || dto.getServiceApi().equals(ServiceApiConstants.CSAT_AGEGROUP) + || dto.getServiceApi().equals(ServiceApiConstants.CSAT_STORE_TIME) + || dto.getServiceApi().equals(ServiceApiConstants.CSAT_TOP_PERFORMING) + || dto.getServiceApi().equals(ServiceApiConstants.CSAT_LEAST_PERFORMING)) { + + entryKey = "storeId"; + } + if (queryMap.containsKey(ElasticProperties.Query.MATCH_CONDITION)) { + Map> queryInnerMap = queryMap + .get(ElasticProperties.Query.MATCH_CONDITION); + queryInnerMap.put(entryKey, valueList); + } else { + Map> queryInnerMap = new HashMap<>(); + queryInnerMap.put(entryKey, valueList); + queryMap.put(ElasticProperties.Query.MATCH_CONDITION, queryInnerMap); + } + } + } + } + } + + if (dto.getDates() != null) { + if (StringUtils.isNotBlank(dto.getDates().getStartDate()) + && StringUtils.isNotBlank(dto.getDates().getEndDate())) { + List valueList = new ArrayList<>(); + + valueList.add(dto.getDates().getStartDate()); + valueList.add(dto.getDates().getEndDate()); + Map> queryInnerMap = new HashMap<>(); + if (dto.getServiceApi().equals(ServiceApiConstants.CSAT_STORE_WRT_TIME) + || dto.getServiceApi().equals(ServiceApiConstants.CSAT_STORE) + || dto.getServiceApi().equals(ServiceApiConstants.CSAT_STORE_REASON) + || dto.getServiceApi().equals(ServiceApiConstants.CSAT_GENDER) + || dto.getServiceApi().equals(ServiceApiConstants.CSAT_AGEGROUP) + || dto.getServiceApi().equals(ServiceApiConstants.CSAT_STORE_TIME) + || dto.getServiceApi().equals(ServiceApiConstants.CSAT_TOP_PERFORMING) + || dto.getServiceApi().equals(ServiceApiConstants.CSAT_LEAST_PERFORMING) + || dto.getServiceApi().equals(ServiceApiConstants.CSAT_DAY_WISE)) { + queryInnerMap.put(ElasticProperties.Query.FEEDBACK_DATE_TIME, valueList); + } else { + queryInnerMap.put(ElasticProperties.Query.TRANSACTION_DATE_FIELD, valueList); + } + + queryMap.put(ElasticProperties.Query.RANGE_CONDITION, queryInnerMap); + } + } + dictator.setQueryMap(queryMap); + + */ +/* + * Map>> aggregationMap = new + * HashMap<>(); Map> innerMap = new + * HashMap<>(); Map deepInnerMap = new HashMap<>(); + *//* + + + return dictator; + + } +*/ + + @SuppressWarnings("unchecked") + @Override + public SearchRequest buildElasticSearchQuery(ElasticSearchDictator dictator) { + SearchSourceBuilder searchBuilder = buildSearchSourceBuilder(); + BoolQueryBuilder boolQuery = buildBoolQuery(); + String localDateStartRange = null; + String localDateEndRange = null; + if (dictator.getQueryMap().containsKey(ElasticProperties.Query.RANGE_CONDITION)) { + + String searchParamRange = null; + Map> innerMap = dictator.getQueryMap().get(ElasticProperties.Query.RANGE_CONDITION); + for (Entry> en : innerMap.entrySet()) { + searchParamRange = en.getKey(); + localDateStartRange = (String) en.getValue().get(0); + localDateEndRange = (String) en.getValue().get(1); + } + + addMustOnBoolQuery(boolQuery, buildRangeQuery(searchParamRange, localDateStartRange, localDateEndRange)); + + } + + if (dictator.getQueryMap().containsKey(ElasticProperties.Query.MATCH_CONDITION)) { + String searchTermField = null; + Map> innerMap = dictator.getQueryMap().get(ElasticProperties.Query.MATCH_CONDITION); + for (Entry> en : innerMap.entrySet()) { + searchTermField = en.getKey(); + addFilterTermsOnBoolQuery(boolQuery, buildTermsQuery(searchTermField, en.getValue())); + } + } + + addQueryToSearchBuilder(searchBuilder, boolQuery); + + DateHistogramAggregationBuilder dateAggBuilder = null; + AvgAggregationBuilder avgAggBuilder = null; + SumAggregationBuilder sumAggBuilder = null; + TermsAggregationBuilder termsAggBuilder = null; + NestedAggregationBuilder nestedAggBuilder = null; + ValueCountAggregationBuilder valueCountAggBuilder = null; + if(dictator.getQueryAggregationMap()==null) { + return new SearchRequest(dictator.getIndexName()).types(dictator.getDocumentType()).source(searchBuilder); + } + + for (Map.Entry itr : dictator.getQueryAggregationMap().entrySet()) { + + String aggregationType = itr.getKey(); + Object value = itr.getValue(); + if (ElasticProperties.Query.NESTED.equals(aggregationType.toUpperCase())) { + Map nestedMap = null; + if (value instanceof HashMap) { + nestedMap = (HashMap) value; + } else if (value instanceof LinkedHashMap) { + nestedMap = (LinkedHashMap) value; + } + String aggregationName = aggregationType + "of" + + nestedMap.get(ElasticProperties.Query.PATH.toLowerCase()); + aggregationName = String.valueOf(nestedMap.get(ElasticProperties.Query.LABEL.toLowerCase())); + nestedAggBuilder = buildNestedAggregationBuilder(aggregationName, nestedMap); + + } else if (ElasticProperties.Query.SUM.equals(aggregationType.toUpperCase())) { + Map sumMap = null; + if (value instanceof HashMap) { + sumMap = (HashMap) value; + } else if (value instanceof LinkedHashMap) { + sumMap = (LinkedHashMap) value; + } + String aggregationName = aggregationType + "of" + + sumMap.get(ElasticProperties.Query.FIELD.toLowerCase()); + aggregationName = String.valueOf(sumMap.get(ElasticProperties.Query.LABEL.toLowerCase())); + sumAggBuilder = buildSumAggregation(aggregationName, + String.valueOf(sumMap.get(ElasticProperties.Query.FIELD.toLowerCase()))); + } else if (ElasticProperties.Query.TERM.equals(aggregationType.toUpperCase())) { + Map termMap = null; + if (value instanceof HashMap) { + termMap = (HashMap) value; + } else if (value instanceof LinkedHashMap) { + termMap = (LinkedHashMap) value; + + } + String aggregationName = aggregationType + "of" + + termMap.get(ElasticProperties.Query.FIELD.toLowerCase()); + aggregationName = String.valueOf(termMap.get(ElasticProperties.Query.LABEL.toLowerCase())); + termsAggBuilder = buildTermAggregation(aggregationName, termMap); + + } else if (ElasticProperties.Query.DATE_HISTOGRAM.equals(aggregationType.toUpperCase())) { + Map histogramMap = null; + + if (value instanceof HashMap) { + histogramMap = (HashMap) value; + } else if (value instanceof LinkedHashMap) { + histogramMap = (LinkedHashMap) value; + + } + String aggregationName = aggregationType + "of" + + histogramMap.get(ElasticProperties.Query.FIELD.toLowerCase()); + aggregationName = String.valueOf(histogramMap.get(ElasticProperties.Query.LABEL.toLowerCase())); + dateAggBuilder = buildDateHistogramAggregation(aggregationName, + histogramMap.get(ElasticProperties.Query.FIELD.toLowerCase()).toString(), + histogramMap.get(ElasticProperties.Query.INTERVAL.toLowerCase()).toString().toUpperCase(), + Long.parseLong(localDateStartRange), Long.parseLong(localDateEndRange)); + + } else if (ElasticProperties.Query.COUNT.equals(aggregationType.toUpperCase())) { + Map countAggMap = null; + + if (value instanceof HashMap) { + countAggMap = (HashMap) value; + } else if (value instanceof LinkedHashMap) { + countAggMap = (LinkedHashMap) value; + + } + String countField = countAggMap.get(ElasticProperties.Query.FIELD.toLowerCase()).toString(); + String countAggName = aggregationType + "of" + countField; + countAggName = String.valueOf(countAggMap.get(ElasticProperties.Query.LABEL.toLowerCase())); + valueCountAggBuilder = buildCountsAggregationQuery(countAggName, countField); + } else if (ElasticProperties.Query.AGGREGATION_CONDITION.equals(aggregationType.toUpperCase())) { + + Map firstLevelAggMap = (LinkedHashMap) value; + for (Map.Entry firstLevelItrEntry : firstLevelAggMap.entrySet()) { + + String firstLevelEntryKey = firstLevelItrEntry.getKey(); + Object firstLevelEntryValue = firstLevelItrEntry.getValue(); + if (ElasticProperties.Query.SUM.equals(firstLevelEntryKey.toUpperCase())) { + Map sumAggMap = (LinkedHashMap) firstLevelEntryValue; + String sumField = sumAggMap.get(ElasticProperties.Query.FIELD.toLowerCase()).toString(); + String sumAggName = firstLevelEntryKey + "of" + sumField; + sumAggName = String.valueOf(sumAggMap.get(ElasticProperties.Query.LABEL.toLowerCase())); + if (termsAggBuilder != null) { + termsAggBuilder = buildSubSumAggreationForTerms(termsAggBuilder, sumAggName, sumField); + } else if (dateAggBuilder != null) { + dateAggBuilder = buildSubSumAggreationForDateHistogram(dateAggBuilder, sumAggName, + sumField); + } + } else if (ElasticProperties.Query.AVG.equals(firstLevelEntryKey.toUpperCase())) { + Map avgAggMap = (LinkedHashMap) firstLevelEntryValue; + String avgField = avgAggMap.get(ElasticProperties.Query.FIELD.toLowerCase()).toString(); + String avgAggName = firstLevelEntryKey + "of" + avgField; + avgAggName = String.valueOf(avgAggMap.get(ElasticProperties.Query.LABEL.toLowerCase())); + if (termsAggBuilder != null) { + termsAggBuilder = buildSubAvgAggreationForTerms(termsAggBuilder, avgAggName, avgField); + } else if (dateAggBuilder != null) { + dateAggBuilder = buildSubSumAggreationForDateHistogram(dateAggBuilder, avgAggName, + avgField); + } + } else if (ElasticProperties.Query.COUNT.equals(firstLevelEntryKey.toUpperCase())) { + Map countAggMap = (LinkedHashMap) firstLevelEntryValue; + String countField = countAggMap.get(ElasticProperties.Query.FIELD.toLowerCase()).toString(); + String countAggName = firstLevelEntryKey + "of" + countField; + countAggName = String.valueOf(countAggMap.get(ElasticProperties.Query.LABEL.toLowerCase())); + if (termsAggBuilder != null) { + termsAggBuilder = buildSubCountAggregationForTerms(termsAggBuilder, countAggName, + countField); + } else if (dateAggBuilder != null) { + dateAggBuilder = buildSubCountAggregationForDateHistogram(dateAggBuilder, countAggName, + countField); + } + } else if (ElasticProperties.Query.TERM.equals(firstLevelEntryKey.toUpperCase())) { + Map subTermAggMap = (LinkedHashMap) firstLevelEntryValue; + String subTermAggName = firstLevelEntryKey + "of" + + subTermAggMap.get(ElasticProperties.Query.FIELD.toLowerCase()).toString(); + subTermAggName = String.valueOf(subTermAggMap.get(ElasticProperties.Query.LABEL.toLowerCase())); + if (nestedAggBuilder != null) { + buildSubTermsAggregationForNested(nestedAggBuilder, subTermAggName, + subTermAggMap.get(ElasticProperties.Query.FIELD.toLowerCase()).toString()); + } else if (dateAggBuilder != null) { + buildSubTermsAggregationForHistogram(dateAggBuilder, subTermAggName, + subTermAggMap.get(ElasticProperties.Query.FIELD.toLowerCase()).toString()); + } else if (termsAggBuilder != null) { + buildSubTermsAggregationForTerms(termsAggBuilder, subTermAggName, + subTermAggMap.get(ElasticProperties.Query.FIELD.toLowerCase()).toString()); + } + + } else if (ElasticProperties.Query.AGGREGATION_CONDITION.equals(firstLevelEntryKey.toUpperCase())) { + + Map secondLevelAggMap = (LinkedHashMap) firstLevelEntryValue; + for (Map.Entry secondLevelItrEntry : secondLevelAggMap.entrySet()) { + String secondLevelItrEntryKey = secondLevelItrEntry.getKey(); + Object secondLevelItrEntryValue = secondLevelItrEntry.getValue(); + if (ElasticProperties.Query.SUM.equals(secondLevelItrEntryKey.toUpperCase())) { + Map subSumAggMap = (LinkedHashMap) secondLevelItrEntryValue; + + String subSumField = subSumAggMap.get(ElasticProperties.Query.FIELD.toLowerCase()) + .toString(); + String subSumAggName = secondLevelItrEntryKey + "of" + subSumField; + subSumAggName = String.valueOf(subSumAggMap.get(ElasticProperties.Query.LABEL.toLowerCase())); + + if (nestedAggBuilder != null) { + List aggBuilders = nestedAggBuilder.getSubAggregations(); + for (AggregationBuilder aggregationBuilder : aggBuilders) { + + if (aggregationBuilder instanceof TermsAggregationBuilder) { + TermsAggregationBuilder subTermAggBuilder = (TermsAggregationBuilder) aggregationBuilder; + subTermAggBuilder = buildSubSumAggreationForTerms(subTermAggBuilder, + subSumAggName, subSumField); + + } + } + } + } else if (ElasticProperties.Query.COUNT.equals(secondLevelItrEntryKey.toUpperCase())) { + Map subCountAggMap = (LinkedHashMap) secondLevelItrEntryValue; + + String subCountField = subCountAggMap.get(ElasticProperties.Query.FIELD.toLowerCase()) + .toString(); + String subCountAggName = secondLevelItrEntryKey + "of" + subCountField; + subCountAggName = String.valueOf(subCountAggMap.get(ElasticProperties.Query.LABEL.toLowerCase())); + + if (dateAggBuilder != null) { + List aggBuilders = dateAggBuilder.getSubAggregations(); + for (AggregationBuilder aggregationBuilder : aggBuilders) { + + if (aggregationBuilder instanceof TermsAggregationBuilder) { + TermsAggregationBuilder subTermAggBuilder = (TermsAggregationBuilder) aggregationBuilder; + subTermAggBuilder = buildSubCountAggregationForTerms(subTermAggBuilder, + subCountAggName, subCountField); + } + } + } + } + } + } + } + } + } + + if (dateAggBuilder != null) { + addAggregationToSearchBuilder(searchBuilder, dateAggBuilder); + } + if (sumAggBuilder != null) { + addSumAggregationToSearchBuilder(searchBuilder, sumAggBuilder); + } + if (avgAggBuilder != null) { + addAvgAggregationToSearchBuilder(searchBuilder, avgAggBuilder); + } + if (valueCountAggBuilder != null) { + addValueCountAggregationToSearchBuilder(searchBuilder, valueCountAggBuilder); + } + if (termsAggBuilder != null) { + addTermsAggregationToSearchBuilder(searchBuilder, termsAggBuilder); + } + if (nestedAggBuilder != null) { + addNestedAggregationToSearchBuilder(searchBuilder, nestedAggBuilder); + } + return new SearchRequest(dictator.getIndexName()).types(dictator.getDocumentType()).source(searchBuilder); + } + + private NestedAggregationBuilder buildSubTermsAggregationForNested(NestedAggregationBuilder nestedAggBuilder, + String subAggregationName, String fieldName) { + return nestedAggBuilder.subAggregation(AggregationBuilders.terms(subAggregationName).field(fieldName)); + + } + + private void addNestedAggregationToSearchBuilder(SearchSourceBuilder searchBuilder, + NestedAggregationBuilder nestedAggBuilder) { + searchBuilder.aggregation(nestedAggBuilder); + } + + private RangeQueryBuilder buildRangeQuery(String fieldName, Object startRange, Object endRange) { + return QueryBuilders.rangeQuery(fieldName).gte(startRange).lte(endRange); + } + + private TermQueryBuilder buildTermQuery(String fieldName, Object fieldValue) { + return QueryBuilders.termQuery(fieldName, fieldValue); + } + + private TermsQueryBuilder buildTermsQuery(String fieldName, List fieldValue) { + return QueryBuilders.termsQuery(fieldName, fieldValue); + } + + private MatchQueryBuilder buildMatchQuery(String fieldName, Object fieldValue) { + return QueryBuilders.matchQuery(fieldName, fieldValue); + } + + private ValueCountAggregationBuilder buildCountsAggregationQuery(String aggregationName, String fieldName) { + return AggregationBuilders.count(aggregationName).field(fieldName); + } + + private SumAggregationBuilder buildSumAggregation(String aggregationName, String fieldName) { + return AggregationBuilders.sum(aggregationName).field(fieldName); + } + + @SuppressWarnings("unchecked") + private TermsAggregationBuilder buildTermAggregation(String aggregationName, Map paramMap) { + TermsAggregationBuilder aggBuilder = AggregationBuilders.terms(aggregationName); + for (Map.Entry param : paramMap.entrySet()) { + if (param.getKey().toUpperCase().equals(ElasticProperties.Query.FIELD)) { + aggBuilder = aggBuilder.field((String) param.getValue()); + } else if (param.getKey().toUpperCase().equals(ElasticProperties.Query.SIZE)) { + aggBuilder = aggBuilder.size((Integer) param.getValue()); + } else if (param.getKey().toUpperCase().equals(ElasticProperties.Query.ORDER)) { + Map keyMap = (HashMap) param.getValue(); + BucketOrder order = null; + for (Map.Entry valueMap : keyMap.entrySet()) { + Map orderMap = (HashMap) valueMap.getValue(); + + String key = valueMap.getKey() + "of" + orderMap.get(ElasticProperties.Query.FIELD.toLowerCase()); + + order = BucketOrder.aggregation(key, + orderMap.get("orderBy").toUpperCase().equals(ElasticProperties.Query.ASC) ? true : false); + } + aggBuilder = aggBuilder.order(order); + } + } + return aggBuilder; + + } + + private AvgAggregationBuilder buildAvgAggregation(String aggregationName, String fieldName) { + return AggregationBuilders.avg(aggregationName).field(fieldName); + } + + private NestedAggregationBuilder buildNestedAggregationBuilder(String aggregationName, + Map paramMap) { + String pathName = paramMap.get(ElasticProperties.Query.PATH.toLowerCase()).toString(); + return AggregationBuilders.nested(aggregationName, pathName); + } + + private AggregationBuilder buildSubTermForNestedAggregation(NestedAggregationBuilder nestedBuilder, + String aggregationName, Map paramMap) { + return nestedBuilder.subAggregation(buildTermAggregation(aggregationName, paramMap)); + + } + + private DateHistogramAggregationBuilder buildDateHistogramAggregation(String aggregationName, String fieldName, + String interval, Long boundMin, Long boundMax) { + DateHistogramInterval dateHistogramInterval = null; + if (ElasticProperties.Query.HOUR.equals(interval)) { + dateHistogramInterval = DateHistogramInterval.HOUR; + } else if (ElasticProperties.Query.DAY.equals(interval)) { + dateHistogramInterval = DateHistogramInterval.DAY; + } else if (ElasticProperties.Query.MINUTE.equals(interval)) { + dateHistogramInterval = DateHistogramInterval.MINUTE; + } else if (ElasticProperties.Query.MONTH.equals(interval)) { + dateHistogramInterval = DateHistogramInterval.MONTH; + + } + return AggregationBuilders.dateHistogram(aggregationName).field(fieldName) + .dateHistogramInterval(dateHistogramInterval).format("epoch_millis") + .extendedBounds(new ExtendedBounds(boundMin, boundMax)); + } + + private DateHistogramAggregationBuilder buildSubSumAggreationForDateHistogram( + DateHistogramAggregationBuilder builder, String aggregationName, String fieldName) { + return builder.subAggregation(AggregationBuilders.sum(aggregationName).field(fieldName)); + } + + private DateHistogramAggregationBuilder buildSubCummulativeSumAggreationForDateHistogram( + DateHistogramAggregationBuilder builder, String aggregationName, String fieldName) { + PipelineAggregationBuilder pipeLineAggregation = new CumulativeSumPipelineAggregationBuilder(aggregationName, + fieldName); + return builder.subAggregation(pipeLineAggregation); + } + + private DateHistogramAggregationBuilder buildSubAvgAggreationForDateHistogram( + DateHistogramAggregationBuilder builder, String aggregationName, String fieldName) { + return builder.subAggregation(AggregationBuilders.avg(aggregationName).field(fieldName)); + } + + private DateHistogramAggregationBuilder buildSubCountAggregationForDateHistogram( + DateHistogramAggregationBuilder builder, String aggregationName, String fieldName) { + return builder.subAggregation(AggregationBuilders.count(aggregationName).field(fieldName)); + } + + private TermsAggregationBuilder buildSubCountAggregationForTerms(TermsAggregationBuilder builder, + String aggregationName, String fieldName) { + return builder.subAggregation(AggregationBuilders.count(aggregationName).field(fieldName)); + } + + private TermsAggregationBuilder buildSubTermsAggregationForTerms(TermsAggregationBuilder builder, + String aggregationName, String fieldName) { + return builder.subAggregation(AggregationBuilders.terms(aggregationName).field(fieldName)); + } + + private DateHistogramAggregationBuilder buildSubTermsAggregationForHistogram( + DateHistogramAggregationBuilder builder, String aggregationName, String fieldName) { + return builder.subAggregation(AggregationBuilders.terms(aggregationName).field(fieldName)); + } + + private TermsAggregationBuilder buildSubSumAggreationForTerms(TermsAggregationBuilder builder, + String aggregationName, String fieldName) { + return builder.subAggregation(AggregationBuilders.sum(aggregationName).field(fieldName)); + } + + private TermsAggregationBuilder buildSubAvgAggreationForTerms(TermsAggregationBuilder builder, + String aggregationName, String fieldName) { + return builder.subAggregation(AggregationBuilders.avg(aggregationName).field(fieldName)); + } + + private SumAggregationBuilder buildSubSumAggregation(SumAggregationBuilder builder, String aggregationName, + String fieldName) { + return builder.subAggregation(AggregationBuilders.sum(aggregationName).field(fieldName)); + } + + private AvgAggregationBuilder buildSubAvgAggregation(AvgAggregationBuilder builder, String aggregationName, + String fieldName) { + return builder.subAggregation(AggregationBuilders.avg(aggregationName).field(fieldName)); + } + + private BoolQueryBuilder buildBoolQuery() { + return QueryBuilders.boolQuery(); + } + + private BoolQueryBuilder addMustOnBoolQuery(BoolQueryBuilder builder, RangeQueryBuilder rangeBuilder) { + return builder.must(rangeBuilder); + } + + private BoolQueryBuilder addFilterOnBoolQuery(BoolQueryBuilder builder, TermQueryBuilder termBuilder) { + return builder.filter(termBuilder); + } + + private BoolQueryBuilder addFilterTermsOnBoolQuery(BoolQueryBuilder builder, TermsQueryBuilder termBuilder) { + return builder.filter(termBuilder); + } + + private BoolQueryBuilder addMatchOnBoolQuery(BoolQueryBuilder builder, MatchQueryBuilder matchBuilder) { + return builder.filter(matchBuilder); + } + + private SearchSourceBuilder buildSearchSourceBuilder() { + return new SearchSourceBuilder().size(0); + } + + private SearchSourceBuilder addQueryToSearchBuilder(SearchSourceBuilder builder, BoolQueryBuilder queryBuilder) { + return builder.query(queryBuilder); + } + + private SearchSourceBuilder addAggregationToSearchBuilder(SearchSourceBuilder builder, + DateHistogramAggregationBuilder aggBuilder) { + return builder.aggregation(aggBuilder); + } + + private SearchSourceBuilder addSumAggregationToSearchBuilder(SearchSourceBuilder builder, + SumAggregationBuilder aggBuilder) { + return builder.aggregation(aggBuilder); + } + + private SearchSourceBuilder addAvgAggregationToSearchBuilder(SearchSourceBuilder builder, + AvgAggregationBuilder aggBuilder) { + return builder.aggregation(aggBuilder); + } + + private SearchSourceBuilder addValueCountAggregationToSearchBuilder(SearchSourceBuilder builder, + ValueCountAggregationBuilder aggBuilder) { + return builder.aggregation(aggBuilder); + } + private SearchSourceBuilder addTermsAggregationToSearchBuilder(SearchSourceBuilder builder, + TermsAggregationBuilder aggBuilder) { + return builder.aggregation(aggBuilder); + } + + @Override + public MultiSearchResponse executeMultiSearchRequest(List searchRequestList, String tenant) { + MultiSearchRequest multiRequest = new MultiSearchRequest(); + MultiSearchResponse response = null; + + for (SearchRequest request : searchRequestList) { + logger.info("ES Query is : " + request.source()); + multiRequest.add(request); + } + + try { + response = client.multiSearch(multiRequest); + } catch (IOException e) { + logger.error("Encountered an error while connecting : " + e); + logger.error("Error Message to report : " + e.getMessage()); + } + + return response; + } + + private RestHighLevelClient getClientForElastic() { + return new RestHighLevelClient(RestClient.builder(new HttpHost(elasticHost, elasticPort, REST_SCHEME))); + } + + private RestHighLevelClient getClientForAlternate() throws MalformedURLException { + URL url = new URL ("https://egov-micro-dev.egovernments.org/elasticsearch/"); + return new RestHighLevelClient(RestClient.builder(new HttpHost(url.getHost(), url.getPort(), REST_SCHEME2))); + } + +/* @SuppressWarnings("unchecked") + @Override + public ElasticSearchDictator createSearchDictator(String indexName, String documentType, + CummulativeDataRequestDto dto, String dateField) throws Exception { + ElasticSearchDictator dictator = new ElasticSearchDictator(); + + dictator.setIndexName(indexName); + dictator.setDocumentType(documentType); + + Map>> queryMap = new HashMap<>(); + if (dto.getCustomData() != null) { + for (Map.Entry entry : dto.getCustomData().entrySet()) { + if (StringUtils.isNotBlank(entry.getKey()) && entry.getValue() != null) { + List valueList = new ArrayList<>(); + + if (entry.getValue() instanceof ArrayList) { + + List valueArray = (ArrayList) entry.getValue(); + + for (Object value : valueArray) { + valueList.add(value); + } + } else { + valueList.add(entry.getValue()); + } + if (!valueList.isEmpty()) { + String entryKey = entry.getKey(); + if (queryMap.containsKey(ElasticProperties.Query.MATCH_CONDITION)) { + Map> queryInnerMap = queryMap + .get(ElasticProperties.Query.MATCH_CONDITION); + queryInnerMap.put(entryKey, valueList); + } else { + Map> queryInnerMap = new HashMap<>(); + queryInnerMap.put(entryKey, valueList); + queryMap.put(ElasticProperties.Query.MATCH_CONDITION, queryInnerMap); + } + } + } + } + } + + if (dto.getDates() != null) { + if (StringUtils.isNotBlank(dto.getDates().getStartDate()) + && StringUtils.isNotBlank(dto.getDates().getEndDate())) { + List valueList = new ArrayList<>(); + + valueList.add(dto.getDates().getStartDate()); + valueList.add(dto.getDates().getEndDate()); + Map> queryInnerMap = new HashMap<>(); + queryInnerMap.put(dateField, valueList); + queryMap.put(ElasticProperties.Query.RANGE_CONDITION, queryInnerMap); + } + } + dictator.setQueryMap(queryMap); + + *//* + * Map>> aggregationMap = new + * HashMap<>(); Map> innerMap = new + * HashMap<>(); Map deepInnerMap = new HashMap<>(); + *//* + + return dictator; + } + + @Override + public ElasticSearchDictator createSearchDictator(AggregateRequestDto dto, String indexName, String documentType, + String filterDateField) throws Exception { + ElasticSearchDictator dictator = new ElasticSearchDictator(); + + dictator.setIndexName(indexName); + dictator.setDocumentType(documentType); + if (StringUtils.isNotBlank(dto.getServiceApi())) { + dictator.setVisualisationName(dto.getServiceApi()); + } + + Map>> queryMap = new HashMap<>(); + if (dto.getCustomData() != null) { + for (Map.Entry entry : dto.getCustomData().entrySet()) { + if (StringUtils.isNotBlank(entry.getKey()) && entry.getValue() != null) { + List valueList = new ArrayList<>(); + + if (entry.getValue() instanceof ArrayList) { + + List valueArray = (ArrayList) entry.getValue(); + + for (Object value : valueArray) { + valueList.add(value); + } + } else { + valueList.add(entry.getValue()); + } + if (!valueList.isEmpty()) { + String entryKey = entry.getKey(); + if (queryMap.containsKey(ElasticProperties.Query.MATCH_CONDITION)) { + Map> queryInnerMap = queryMap + .get(ElasticProperties.Query.MATCH_CONDITION); + queryInnerMap.put(entryKey, valueList); + } else { + Map> queryInnerMap = new HashMap<>(); + queryInnerMap.put(entryKey, valueList); + queryMap.put(ElasticProperties.Query.MATCH_CONDITION, queryInnerMap); + } + } + } + } + } + + if (dto.getDates() != null) { + if (StringUtils.isNotBlank(dto.getDates().getStartDate()) + && StringUtils.isNotBlank(dto.getDates().getEndDate())) { + List valueList = new ArrayList<>(); + + valueList.add(dto.getDates().getStartDate()); + valueList.add(dto.getDates().getEndDate()); + Map> queryInnerMap = new HashMap<>(); + + queryInnerMap.put(filterDateField, valueList); + + queryMap.put(ElasticProperties.Query.RANGE_CONDITION, queryInnerMap); + } + } + dictator.setQueryMap(queryMap); + return dictator; + }*/ + + @Override + public ElasticSearchDictator createSearchDictatorV2(AggregateRequestDto dto, String indexName, String documentType, + String filterDateField) throws Exception { + ElasticSearchDictator dictator = new ElasticSearchDictator(); + + dictator.setIndexName(indexName); + dictator.setDocumentType(documentType); + + Map>> queryMap = new HashMap<>(); + if (dto.getEsFilters() != null && !dto.getEsFilters().isEmpty()) { + for (Map.Entry entry : dto.getEsFilters().entrySet()) { + if (StringUtils.isNotBlank(entry.getKey()) && entry.getValue() != null) { + List valueList = new ArrayList<>(); + + if (entry.getValue() instanceof ArrayList) { + + List valueArray = (ArrayList) entry.getValue(); + + for (Object value : valueArray) { + String valueString = ""; + if(entry.getKey().equals("dataObject.tenantId")) { + valueString = String.valueOf(value); + valueString = valueString.replace("uat: ", ""); + } + if(StringUtils.isBlank(valueString)) valueList.add(value); + else valueList.add(valueString); + } + } else { + String valueString = ""; + if(entry.getKey().equals("dataObject.tenantId")) { + valueString = String.valueOf(entry.getValue()); + valueString = valueString.replace("uat: ", ""); + } + if(StringUtils.isBlank(valueString)) valueList.add(entry.getValue()); + else valueList.add(valueString); + } + if (!valueList.isEmpty()) { + String entryKey = entry.getKey(); + if (queryMap.containsKey(ElasticProperties.Query.MATCH_CONDITION)) { + Map> queryInnerMap = queryMap + .get(ElasticProperties.Query.MATCH_CONDITION); + queryInnerMap.put(entryKey, valueList); + } else { + Map> queryInnerMap = new HashMap<>(); + queryInnerMap.put(entryKey, valueList); + queryMap.put(ElasticProperties.Query.MATCH_CONDITION, queryInnerMap); + } + } + } + } + } + + Map queryAggregationMap = new HashMap<>(); + if (dto.getAggregationFactors() != null && !dto.getAggregationFactors().isEmpty()) { + for (Map.Entry entry : dto.getAggregationFactors().entrySet()) { + if (StringUtils.isNotBlank(entry.getKey()) && entry.getValue() != null) { + Map innerAggregationMap = new HashMap<>(); + innerAggregationMap.put("field", entry.getValue()); + queryAggregationMap.put(entry.getKey(), innerAggregationMap); + } + } + } + dictator.setQueryAggregationMap(queryAggregationMap); + if (dto.getRequestDate() != null) { + if (StringUtils.isNotBlank(dto.getRequestDate().getStartDate()) + && StringUtils.isNotBlank(dto.getRequestDate().getEndDate())) { + List valueList = new ArrayList<>(); + + valueList.add(dto.getRequestDate().getStartDate()); + valueList.add(dto.getRequestDate().getEndDate()); + Map> queryInnerMap = new HashMap<>(); + + if(StringUtils.isNotBlank(filterDateField)) { + queryInnerMap.put(filterDateField, valueList); + queryMap.put(ElasticProperties.Query.RANGE_CONDITION, queryInnerMap); + } + } + } + dictator.setQueryMap(queryMap); + return dictator; + } + + @Override + public MultiSearchResponse executeMultiSearchRequest(List searchRequestList, Boolean primaryOrNot) { + MultiSearchRequest multiRequest = new MultiSearchRequest(); + MultiSearchResponse response = null; + for (SearchRequest request : searchRequestList) { + logger.info("ES Query is : " + request.source()); + multiRequest.add(request); + } + + try { + if(primaryOrNot) { + response = client.multiSearch(multiRequest); + } else { + logger.info("Alternate Client URL : " + alternateClient.toString()); + response = alternateClient.multiSearch(multiRequest); + } + + } catch (IOException e) { + logger.error("Encountered an error while connecting : " + e); + logger.error("Error Message to report : " + e.getMessage()); + } + return response; + } +} diff --git a/dashboard-analytics/src/main/java/com/tarento/analytics/dto/AggregateDto.java b/dashboard-analytics/src/main/java/com/tarento/analytics/dto/AggregateDto.java new file mode 100644 index 000000000..09f225ebc --- /dev/null +++ b/dashboard-analytics/src/main/java/com/tarento/analytics/dto/AggregateDto.java @@ -0,0 +1,95 @@ +package com.tarento.analytics.dto; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import com.tarento.analytics.enums.ChartType; + +/** + * @author Darshan Nagesh + * + */ +public class AggregateDto { + + private ChartType chartType; + + private String visualizationCode; + + private String chartFormat; + + private String drillDownChartId; + + public String getVisualizationCode() { + return visualizationCode; + } + + public void setVisualizationCode(String visualizationCode) { + this.visualizationCode = visualizationCode; + } + + public String getDrillDownChartId() { + return drillDownChartId; + } + + public void setDrillDownChartId(String drillDownChartId) { + this.drillDownChartId = drillDownChartId; + } + + private Map customData; + + private RequestDate dates; + + private Object filter; + + private List data = new ArrayList<>(); + + public List getData() { + return data; + } + + public void setData(List data) { + this.data = data; + } + + public ChartType getChartType() { + return chartType; + } + + public void setChartType(ChartType chartType) { + this.chartType = chartType; + } + + public String getChartFormat() { + return chartFormat; + } + + public void setChartFormat(String chartFormat) { + this.chartFormat = chartFormat; + } + + + public Map getCustomData() { + return customData; + } + + public void setCustomData(Map customData) { + this.customData = customData; + } + + public RequestDate getDates() { + return dates; + } + + public void setDates(RequestDate dates) { + this.dates = dates; + } + + public Object getFilter() { + return filter; + } + + public void setFilter(Object filter) { + this.filter = filter; + } +} diff --git a/dashboard-analytics/src/main/java/com/tarento/analytics/dto/AggregateRequestDto.java b/dashboard-analytics/src/main/java/com/tarento/analytics/dto/AggregateRequestDto.java new file mode 100644 index 000000000..0459fd6c7 --- /dev/null +++ b/dashboard-analytics/src/main/java/com/tarento/analytics/dto/AggregateRequestDto.java @@ -0,0 +1,108 @@ +package com.tarento.analytics.dto; + +import com.fasterxml.jackson.databind.node.ObjectNode; + +import java.util.Map; + +public class AggregateRequestDto { + + private String requestId; + private String visualizationType; + private String visualizationCode; + private String moduleLevel; + private String queryType; + private Map filters; + private Map esFilters; + private Map aggregationFactors; + private RequestDate requestDate; + private String interval; + private ObjectNode chartNode; + + public AggregateRequestDto() {} + public AggregateRequestDto(AggregateRequestDtoV3 requestDtoV3, String visualizationType, String visualizationCode) { + this.visualizationCode = visualizationCode; + this.visualizationType = visualizationType; + this.moduleLevel = requestDtoV3.getModuleLevel(); + this.queryType = requestDtoV3.getQueryType(); + this.filters = requestDtoV3.getFilters(); + this.esFilters = requestDtoV3.getEsFilters(); + this.aggregationFactors = requestDtoV3.getAggregationFactors(); + this.requestDate = requestDtoV3.getRequestDate(); + this.interval = requestDtoV3.getInterval(); + this.chartNode = requestDtoV3.getChartNode(); + this.requestId= requestDtoV3.getRequestId(); + } + + + public String getRequestId() { + return requestId; + } + public void setRequestId(String requestId) { + this.requestId = requestId; + } + public ObjectNode getChartNode() { + return chartNode; + } + + public void setChartNode(ObjectNode chartNode) { + this.chartNode = chartNode; + } + + public String getModuleLevel() { + return moduleLevel; + } + public void setModuleLevel(String moduleLevel) { + this.moduleLevel = moduleLevel; + } + public Map getEsFilters() { + return esFilters; + } + public void setEsFilters(Map esFilters) { + this.esFilters = esFilters; + } + public String getVisualizationCode() { + return visualizationCode; + } + public void setVisualizationCode(String visualizationCode) { + this.visualizationCode = visualizationCode; + } + public String getVisualizationType() { + return visualizationType; + } + public void setVisualizationType(String visualizationType) { + this.visualizationType = visualizationType; + } + public String getQueryType() { + return queryType; + } + public void setQueryType(String queryType) { + this.queryType = queryType; + } + public Map getFilters() { + return filters; + } + public void setFilters(Map filters) { + this.filters = filters; + } + public Map getAggregationFactors() { + return aggregationFactors; + } + public void setAggregationFactors(Map aggregationFactors) { + this.aggregationFactors = aggregationFactors; + } + public RequestDate getRequestDate() { + return requestDate; + } + public void setRequestDate(RequestDate requestDate) { + this.requestDate = requestDate; + } + public String getInterval() { + return interval; + } + public void setInterval(String interval) { + this.interval = interval; + } + + + +} diff --git a/dashboard-analytics/src/main/java/com/tarento/analytics/dto/AggregateRequestDtoV3.java b/dashboard-analytics/src/main/java/com/tarento/analytics/dto/AggregateRequestDtoV3.java new file mode 100644 index 000000000..13b3a31f1 --- /dev/null +++ b/dashboard-analytics/src/main/java/com/tarento/analytics/dto/AggregateRequestDtoV3.java @@ -0,0 +1,86 @@ +package com.tarento.analytics.dto; + +import java.util.List; +import java.util.Map; + +import com.fasterxml.jackson.databind.node.ObjectNode; + +public class AggregateRequestDtoV3 { + + private String requestId; + private List visualizations; + private String moduleLevel; + private String queryType; + private Map filters; + private Map esFilters; + private Map aggregationFactors; + private RequestDate requestDate; + private String interval; + private ObjectNode chartNode; + + public ObjectNode getChartNode() { + return chartNode; + } + + public void setChartNode(ObjectNode chartNode) { + this.chartNode = chartNode; + } + public String getRequestId() { + return requestId; + } + + public void setRequestId(String requestId) { + this.requestId = requestId; + } + public String getModuleLevel() { + return moduleLevel; + } + public void setModuleLevel(String moduleLevel) { + this.moduleLevel = moduleLevel; + } + public Map getEsFilters() { + return esFilters; + } + public void setEsFilters(Map esFilters) { + this.esFilters = esFilters; + } + public String getQueryType() { + return queryType; + } + public void setQueryType(String queryType) { + this.queryType = queryType; + } + public Map getFilters() { + return filters; + } + public void setFilters(Map filters) { + this.filters = filters; + } + public Map getAggregationFactors() { + return aggregationFactors; + } + public void setAggregationFactors(Map aggregationFactors) { + this.aggregationFactors = aggregationFactors; + } + public RequestDate getRequestDate() { + return requestDate; + } + public void setRequestDate(RequestDate requestDate) { + this.requestDate = requestDate; + } + public String getInterval() { + return interval; + } + public void setInterval(String interval) { + this.interval = interval; + } + + public List getVisualizations() { + return visualizations; + } + + public void setVisualizations(List visualizations) { + this.visualizations = visualizations; + } + +} diff --git a/dashboard-analytics/src/main/java/com/tarento/analytics/dto/City.java b/dashboard-analytics/src/main/java/com/tarento/analytics/dto/City.java new file mode 100644 index 000000000..f436509ef --- /dev/null +++ b/dashboard-analytics/src/main/java/com/tarento/analytics/dto/City.java @@ -0,0 +1,186 @@ +package com.tarento.analytics.dto; + +import java.util.HashMap; +import java.util.Map; +import com.fasterxml.jackson.annotation.JsonAnyGetter; +import com.fasterxml.jackson.annotation.JsonAnySetter; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonPropertyOrder; + +@JsonInclude(JsonInclude.Include.NON_NULL) +@JsonPropertyOrder({ "name", "localName", "districtCode", "districtName", "regionName", "ulbGrade", "longitude", + "latitude", "shapeFileLocation", "captcha", "code", "regionCode", "municipalityName" }) +public class City { + + @JsonProperty("name") + private String name; + @JsonProperty("localName") + private String localName; + @JsonProperty("districtCode") + private String districtCode; + @JsonProperty("districtName") + private String districtName; + @JsonProperty("regionName") + private String regionName; + @JsonProperty("ulbGrade") + private String ulbGrade; + @JsonProperty("longitude") + private Double longitude; + @JsonProperty("latitude") + private Double latitude; + @JsonProperty("shapeFileLocation") + private Object shapeFileLocation; + @JsonProperty("captcha") + private Object captcha; + @JsonProperty("code") + private String code; + @JsonProperty("regionCode") + private String regionCode; + @JsonProperty("municipalityName") + private String municipalityName; + @JsonIgnore + private Map additionalProperties = new HashMap(); + + @JsonProperty("name") + public String getName() { + return name; + } + + @JsonProperty("name") + public void setName(String name) { + this.name = name; + } + + @JsonProperty("localName") + public String getLocalName() { + return localName; + } + + @JsonProperty("localName") + public void setLocalName(String localName) { + this.localName = localName; + } + + @JsonProperty("districtCode") + public String getDistrictCode() { + return districtCode; + } + + @JsonProperty("districtCode") + public void setDistrictCode(String districtCode) { + this.districtCode = districtCode; + } + + @JsonProperty("districtName") + public String getDistrictName() { + return districtName; + } + + @JsonProperty("districtName") + public void setDistrictName(String districtName) { + this.districtName = districtName; + } + + @JsonProperty("regionName") + public String getRegionName() { + return regionName; + } + + @JsonProperty("regionName") + public void setRegionName(String regionName) { + this.regionName = regionName; + } + + @JsonProperty("ulbGrade") + public String getUlbGrade() { + return ulbGrade; + } + + @JsonProperty("ulbGrade") + public void setUlbGrade(String ulbGrade) { + this.ulbGrade = ulbGrade; + } + + @JsonProperty("longitude") + public Double getLongitude() { + return longitude; + } + + @JsonProperty("longitude") + public void setLongitude(Double longitude) { + this.longitude = longitude; + } + + @JsonProperty("latitude") + public Double getLatitude() { + return latitude; + } + + @JsonProperty("latitude") + public void setLatitude(Double latitude) { + this.latitude = latitude; + } + + @JsonProperty("shapeFileLocation") + public Object getShapeFileLocation() { + return shapeFileLocation; + } + + @JsonProperty("shapeFileLocation") + public void setShapeFileLocation(Object shapeFileLocation) { + this.shapeFileLocation = shapeFileLocation; + } + + @JsonProperty("captcha") + public Object getCaptcha() { + return captcha; + } + + @JsonProperty("captcha") + public void setCaptcha(Object captcha) { + this.captcha = captcha; + } + + @JsonProperty("code") + public String getCode() { + return code; + } + + @JsonProperty("code") + public void setCode(String code) { + this.code = code; + } + + @JsonProperty("regionCode") + public String getRegionCode() { + return regionCode; + } + + @JsonProperty("regionCode") + public void setRegionCode(String regionCode) { + this.regionCode = regionCode; + } + + @JsonProperty("municipalityName") + public String getMunicipalityName() { + return municipalityName; + } + + @JsonProperty("municipalityName") + public void setMunicipalityName(String municipalityName) { + this.municipalityName = municipalityName; + } + + @JsonAnyGetter + public Map getAdditionalProperties() { + return this.additionalProperties; + } + + @JsonAnySetter + public void setAdditionalProperty(String name, Object value) { + this.additionalProperties.put(name, value); + } + +} \ No newline at end of file diff --git a/dashboard-analytics/src/main/java/com/tarento/analytics/dto/CummulativeDataRequestDto.java b/dashboard-analytics/src/main/java/com/tarento/analytics/dto/CummulativeDataRequestDto.java new file mode 100644 index 000000000..3ba4bf8e7 --- /dev/null +++ b/dashboard-analytics/src/main/java/com/tarento/analytics/dto/CummulativeDataRequestDto.java @@ -0,0 +1,32 @@ +package com.tarento.analytics.dto; + +import java.util.List; +import java.util.Map; + +public class CummulativeDataRequestDto { + private Map customData; + private RequestDate dates; + private String dashCode; + + + + public String getDashCode() { + return dashCode; + } + public void setDashCode(String dashCode) { + this.dashCode = dashCode; + } + public Map getCustomData() { + return customData; + } + public void setCustomData(Map customData) { + this.customData = customData; + } + public RequestDate getDates() { + return dates; + } + public void setDates(RequestDate dates) { + this.dates = dates; + } + +} diff --git a/dashboard-analytics/src/main/java/com/tarento/analytics/dto/DashboardDto.java b/dashboard-analytics/src/main/java/com/tarento/analytics/dto/DashboardDto.java new file mode 100644 index 000000000..1ff82a8da --- /dev/null +++ b/dashboard-analytics/src/main/java/com/tarento/analytics/dto/DashboardDto.java @@ -0,0 +1,44 @@ +package com.tarento.analytics.dto; + +public class DashboardDto { + + private Long id; + private String name; + private String code; + private String description; + private String placement; + + public String getPlacement() { + return placement; + } + public void setPlacement(String placement) { + this.placement = placement; + } + public Long getId() { + return id; + } + public void setId(Long id) { + this.id = id; + } + public String getName() { + return name; + } + public void setName(String name) { + this.name = name; + } + public String getCode() { + return code; + } + public void setCode(String code) { + this.code = code; + } + public String getDescription() { + return description; + } + public void setDescription(String description) { + this.description = description; + } + + + +} diff --git a/dashboard-analytics/src/main/java/com/tarento/analytics/dto/DashboardHeaderDto.java b/dashboard-analytics/src/main/java/com/tarento/analytics/dto/DashboardHeaderDto.java new file mode 100644 index 000000000..33db8b1a2 --- /dev/null +++ b/dashboard-analytics/src/main/java/com/tarento/analytics/dto/DashboardHeaderDto.java @@ -0,0 +1,28 @@ +package com.tarento.analytics.dto; + +import com.tarento.analytics.model.LineData; + +public class DashboardHeaderDto { + + private String type; + private Object data; + public String getType() { + return type; + } + public void setType(String type) { + this.type = type; + } + public Object getData() { + return data; + } + public void setData(Object data) { + this.data = data; + } + public LineData getLineData() { + return lineData; + } + public void setLineData(LineData lineData) { + this.lineData = lineData; + } + private LineData lineData; +} diff --git a/dashboard-analytics/src/main/java/com/tarento/analytics/dto/Data.java b/dashboard-analytics/src/main/java/com/tarento/analytics/dto/Data.java new file mode 100644 index 000000000..2673cbf75 --- /dev/null +++ b/dashboard-analytics/src/main/java/com/tarento/analytics/dto/Data.java @@ -0,0 +1,71 @@ +package com.tarento.analytics.dto; + +import java.util.ArrayList; +import java.util.List; + +public class Data { + + private String headerName; + private Object headerValue; + private String headerSymbol; + private InsightsWidget insight; + + public InsightsWidget getInsight() { + return insight; + } + + public void setInsight(InsightsWidget insight) { + this.insight = insight; + } + + public void setHeaderName(String headerName) { + this.headerName = headerName; + } + + public void setHeaderSymbol(String headerSymbol) { + this.headerSymbol = headerSymbol; + } + + + public Data(String name, Object value, String symbol) { + this.headerName = name; + this.headerValue = value; + this.headerSymbol = symbol; + } + + public Data(String name, Object value, String symbol, List plots) { + this.headerName = name; + this.headerValue = value; + this.headerSymbol = symbol; + this.plots = plots; + } + + private List plots = new ArrayList<>(); + + public List getPlots() { + return plots; + } + + public void setPlots(List plots) { + this.plots = plots; + } + + public void setHeaderValue(Object headerValue) { + this.headerValue = headerValue; + } + + public String getHeaderName() { + return headerName; + } + + public Object getHeaderValue() { + return headerValue; + } + + public String getHeaderSymbol() { + return headerSymbol; + } + + + +} diff --git a/dashboard-analytics/src/main/java/com/tarento/analytics/dto/InsightsWidget.java b/dashboard-analytics/src/main/java/com/tarento/analytics/dto/InsightsWidget.java new file mode 100644 index 000000000..0eed70f97 --- /dev/null +++ b/dashboard-analytics/src/main/java/com/tarento/analytics/dto/InsightsWidget.java @@ -0,0 +1,44 @@ +package com.tarento.analytics.dto; + +public class InsightsWidget { + + private String name; + private Object value; + private String indicator; + private String colorCode; + + public InsightsWidget() {} + public InsightsWidget(String name, Object value, String indicator, String colorCode) { + this.name = name; + this.value = value; + this.indicator = indicator; + this.colorCode = colorCode; + } + public String getName() { + return name; + } + public void setName(String name) { + this.name = name; + } + public Object getValue() { + return value; + } + public void setValue(Object value) { + this.value = value; + } + public String getIndicator() { + return indicator; + } + public void setIndicator(String indicator) { + this.indicator = indicator; + } + public String getColorCode() { + return colorCode; + } + public void setColorCode(String colorCode) { + this.colorCode = colorCode; + } + + + +} diff --git a/dashboard-analytics/src/main/java/com/tarento/analytics/dto/MappingDto.java b/dashboard-analytics/src/main/java/com/tarento/analytics/dto/MappingDto.java new file mode 100644 index 000000000..1bb98d2d1 --- /dev/null +++ b/dashboard-analytics/src/main/java/com/tarento/analytics/dto/MappingDto.java @@ -0,0 +1,14 @@ +package com.tarento.analytics.dto; + +public class MappingDto { + + private Object mappings; + + public Object getMappings() { + return mappings; + } + + public void setMappings(Object mappings) { + this.mappings = mappings; + } +} diff --git a/dashboard-analytics/src/main/java/com/tarento/analytics/dto/Plot.java b/dashboard-analytics/src/main/java/com/tarento/analytics/dto/Plot.java new file mode 100644 index 000000000..05b7e0ec6 --- /dev/null +++ b/dashboard-analytics/src/main/java/com/tarento/analytics/dto/Plot.java @@ -0,0 +1,40 @@ +package com.tarento.analytics.dto; + +import java.math.BigDecimal; + +public class Plot { + + private String label; + private String name; + private Double value; + private String symbol; + + public Plot(String name, Double value, String symbol) { + this.name = name; + this.value = value; + this.symbol = symbol; + } + + public String getName() { + return name; + } + + public Double getValue() { + return value; + } + + public String getSymbol() { + return symbol; + } + + public String getLabel() { + return label; + } + + public void setLabel(String label) { + this.label = label; + } + public void setValue(Double value) { + this.value = value; + } +} diff --git a/dashboard-analytics/src/main/java/com/tarento/analytics/dto/RequestDate.java b/dashboard-analytics/src/main/java/com/tarento/analytics/dto/RequestDate.java new file mode 100644 index 000000000..3e615493a --- /dev/null +++ b/dashboard-analytics/src/main/java/com/tarento/analytics/dto/RequestDate.java @@ -0,0 +1,33 @@ +package com.tarento.analytics.dto; + +public class RequestDate { + private String targetDate; + private String startDate; + private String endDate; + private String interval; + public String getInterval() { + return interval; + } + public void setInterval(String interval) { + this.interval = interval; + } + public String getTargetDate() { + return targetDate; + } + public void setTargetDate(String targetDate) { + this.targetDate = targetDate; + } + public String getStartDate() { + return startDate; + } + public void setStartDate(String startDate) { + this.startDate = startDate; + } + public String getEndDate() { + return endDate; + } + public void setEndDate(String endDate) { + this.endDate = endDate; + } + +} diff --git a/dashboard-analytics/src/main/java/com/tarento/analytics/dto/RequestDto.java b/dashboard-analytics/src/main/java/com/tarento/analytics/dto/RequestDto.java new file mode 100644 index 000000000..02e4b86f2 --- /dev/null +++ b/dashboard-analytics/src/main/java/com/tarento/analytics/dto/RequestDto.java @@ -0,0 +1,22 @@ +package com.tarento.analytics.dto; + +import java.util.Map; + +public class RequestDto { + + private Map headers; + private AggregateRequestDto aggregationRequestDto; + + public Map getHeaders() { + return headers; + } + public void setHeaders(Map headers) { + this.headers = headers; + } + public AggregateRequestDto getAggregationRequestDto() { + return aggregationRequestDto; + } + public void setAggregationRequestDto(AggregateRequestDto aggregationRequestDto) { + this.aggregationRequestDto = aggregationRequestDto; + } +} diff --git a/dashboard-analytics/src/main/java/com/tarento/analytics/dto/RequestDtoV3.java b/dashboard-analytics/src/main/java/com/tarento/analytics/dto/RequestDtoV3.java new file mode 100644 index 000000000..f15238402 --- /dev/null +++ b/dashboard-analytics/src/main/java/com/tarento/analytics/dto/RequestDtoV3.java @@ -0,0 +1,22 @@ +package com.tarento.analytics.dto; + +import java.util.Map; + +public class RequestDtoV3 { + + private Map headers; + private AggregateRequestDtoV3 aggregationRequestDto; + + public Map getHeaders() { + return headers; + } + public void setHeaders(Map headers) { + this.headers = headers; + } + public AggregateRequestDtoV3 getAggregationRequestDto() { + return aggregationRequestDto; + } + public void setAggregationRequestDto(AggregateRequestDtoV3 aggregationRequestDto) { + this.aggregationRequestDto = aggregationRequestDto; + } +} diff --git a/dashboard-analytics/src/main/java/com/tarento/analytics/dto/ResponseDto.java b/dashboard-analytics/src/main/java/com/tarento/analytics/dto/ResponseDto.java new file mode 100644 index 000000000..13ac13569 --- /dev/null +++ b/dashboard-analytics/src/main/java/com/tarento/analytics/dto/ResponseDto.java @@ -0,0 +1,40 @@ +package com.tarento.analytics.dto; + +import org.springframework.http.HttpStatus; + +public class ResponseDto { + + private int statusCode; + private String statusMessage; + private Object response; + + public ResponseDto(int statusCode, String statusMessage, Object response) { + this.statusCode = statusCode; + this.statusMessage = statusMessage; + this.response = response; + } + + public int getStatusCode() { + return statusCode; + } + + public void setStatusCode(int statusCode) { + this.statusCode = statusCode; + } + + public String getStatusMessage() { + return statusMessage; + } + + public void setStatusMessage(String statusMessage) { + this.statusMessage = statusMessage; + } + + public Object getResponse() { + return response; + } + + public void setResponse(Object response) { + this.response = response; + } +} diff --git a/dashboard-analytics/src/main/java/com/tarento/analytics/dto/RoleDto.java b/dashboard-analytics/src/main/java/com/tarento/analytics/dto/RoleDto.java new file mode 100644 index 000000000..85fee2ee5 --- /dev/null +++ b/dashboard-analytics/src/main/java/com/tarento/analytics/dto/RoleDto.java @@ -0,0 +1,46 @@ +package com.tarento.analytics.dto; + +public class RoleDto { + + public Long id; + + public String description; + + public String name; + + public String orgId; + + public boolean isSuperAdmin; + + public String getOrgId() { + return orgId; + } + + public void setOrgId(String orgId) { + this.orgId = orgId; + } + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } +} diff --git a/dashboard-analytics/src/main/java/com/tarento/analytics/dto/SearchDto.java b/dashboard-analytics/src/main/java/com/tarento/analytics/dto/SearchDto.java new file mode 100644 index 000000000..14687f398 --- /dev/null +++ b/dashboard-analytics/src/main/java/com/tarento/analytics/dto/SearchDto.java @@ -0,0 +1,165 @@ +package com.tarento.analytics.dto; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * This class will take input for elastic search query + * + * @author + */ +public class SearchDto { + + @SuppressWarnings("rawtypes") + private List properties; + + private List> facets = new ArrayList<>(); + private List fields; + private List excludedFields; + private Map sortBy = new HashMap<>(); + private String operation; + private String query; + private Integer limit = 250; + private Integer offset = 0; + private boolean fuzzySearch = false; + // additional properties will hold , filters, exist , not exist + private Map additionalProperties = new HashMap<>(); + private Map softConstraints = new HashMap<>(); + private List> groupQuery = new ArrayList<>(); + private List mode = new ArrayList<>(); + + public List> getGroupQuery() { + return groupQuery; + } + + public void setGroupQuery(List> groupQuery) { + this.groupQuery = groupQuery; + } + + public SearchDto() { + super(); + } + + @SuppressWarnings("rawtypes") + public SearchDto(List properties, String operation, int limit) { + super(); + this.properties = properties; + this.operation = operation; + this.limit = limit; + } + + @SuppressWarnings("rawtypes") + public List getProperties() { + return properties; + } + + @SuppressWarnings("rawtypes") + public void setProperties(List properties) { + this.properties = properties; + } + + public String getOperation() { + return operation; + } + + public void setOperation(String operation) { + this.operation = operation; + } + + public Integer getLimit() { + return limit; + } + + public void setLimit(Integer limit) { + this.limit = limit; + } + + public List> getFacets() { + return facets; + } + + public void setFacets(List> facets) { + this.facets = facets; + } + + public Map getSortBy() { + return sortBy; + } + + public void setSortBy(Map sortBy) { + this.sortBy = sortBy; + } + + public boolean isFuzzySearch() { + return fuzzySearch; + } + + public void setFuzzySearch(boolean fuzzySearch) { + this.fuzzySearch = fuzzySearch; + } + + public Map getAdditionalProperties() { + return additionalProperties; + } + + public void setAdditionalProperties(Map additionalProperties) { + this.additionalProperties = additionalProperties; + } + + public Object getAdditionalProperty(String key) { + return additionalProperties.get(key); + } + + public void addAdditionalProperty(String key, Object value) { + this.additionalProperties.put(key, value); + } + + public List getFields() { + return fields; + } + + public void setFields(List fields) { + this.fields = fields; + } + + public Integer getOffset() { + return offset; + } + + public void setOffset(Integer offset) { + this.offset = offset; + } + + public Map getSoftConstraints() { + return softConstraints; + } + + public void setSoftConstraints(Map softConstraints) { + this.softConstraints = softConstraints; + } + + public String getQuery() { + return query; + } + + public void setQuery(String query) { + this.query = query; + } + + public List getMode() { + return mode; + } + + public void setMode(List mode) { + this.mode = mode; + } + + public List getExcludedFields() { + return excludedFields; + } + + public void setExcludedFields(List excludedFields) { + this.excludedFields = excludedFields; + } +} \ No newline at end of file diff --git a/dashboard-analytics/src/main/java/com/tarento/analytics/dto/Tenant.java b/dashboard-analytics/src/main/java/com/tarento/analytics/dto/Tenant.java new file mode 100644 index 000000000..a76e519ee --- /dev/null +++ b/dashboard-analytics/src/main/java/com/tarento/analytics/dto/Tenant.java @@ -0,0 +1,206 @@ +package com.tarento.analytics.dto; + +import java.util.HashMap; +import java.util.Map; + +import com.fasterxml.jackson.annotation.JsonAnyGetter; +import com.fasterxml.jackson.annotation.JsonAnySetter; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonPropertyOrder; + +@JsonInclude(JsonInclude.Include.NON_NULL) +@JsonPropertyOrder({ "code", "name", "description", "logoId", "imageId", "domainUrl", "type", "twitterUrl", + "facebookUrl", "emailId", "OfficeTimings", "city", "address", "contactNumber" }) +public class Tenant { + + @JsonProperty("code") + private String code; + @JsonProperty("name") + private String name; + @JsonProperty("description") + private String description; + @JsonProperty("logoId") + private String logoId; + @JsonProperty("imageId") + private Object imageId; + @JsonProperty("domainUrl") + private String domainUrl; + @JsonProperty("type") + private String type; + @JsonProperty("twitterUrl") + private Object twitterUrl; + @JsonProperty("facebookUrl") + private Object facebookUrl; + @JsonProperty("emailId") + private String emailId; + @JsonProperty("city") + private City city; + @JsonProperty("address") + private String address; + @JsonProperty("contactNumber") + private String contactNumber; + @JsonIgnore + private Map additionalProperties = new HashMap(); + + @JsonProperty("code") + public String getCode() { + return code; + } + + @JsonProperty("code") + public void setCode(String code) { + this.code = code; + } + + @JsonProperty("name") + public String getName() { + return name; + } + + @JsonProperty("name") + public void setName(String name) { + this.name = name; + } + + @JsonProperty("description") + public String getDescription() { + return description; + } + + @JsonProperty("description") + public void setDescription(String description) { + this.description = description; + } + + @JsonProperty("logoId") + public String getLogoId() { + return logoId; + } + + @JsonProperty("logoId") + public void setLogoId(String logoId) { + this.logoId = logoId; + } + + @JsonProperty("imageId") + public Object getImageId() { + return imageId; + } + + @JsonProperty("imageId") + public void setImageId(Object imageId) { + this.imageId = imageId; + } + + @JsonProperty("domainUrl") + public String getDomainUrl() { + return domainUrl; + } + + @JsonProperty("domainUrl") + public void setDomainUrl(String domainUrl) { + this.domainUrl = domainUrl; + } + + @JsonProperty("type") + public String getType() { + return type; + } + + @JsonProperty("type") + public void setType(String type) { + this.type = type; + } + + @JsonProperty("twitterUrl") + public Object getTwitterUrl() { + return twitterUrl; + } + + @JsonProperty("twitterUrl") + public void setTwitterUrl(Object twitterUrl) { + this.twitterUrl = twitterUrl; + } + + @JsonProperty("facebookUrl") + public Object getFacebookUrl() { + return facebookUrl; + } + + @JsonProperty("facebookUrl") + public void setFacebookUrl(Object facebookUrl) { + this.facebookUrl = facebookUrl; + } + + @JsonProperty("emailId") + public String getEmailId() { + return emailId; + } + + @JsonProperty("emailId") + public void setEmailId(String emailId) { + this.emailId = emailId; + } + + @JsonProperty("city") + public City getCity() { + return city; + } + + @JsonProperty("city") + public void setCity(City city) { + this.city = city; + } + + @JsonProperty("address") + public String getAddress() { + return address; + } + + @JsonProperty("address") + public void setAddress(String address) { + this.address = address; + } + + @JsonProperty("contactNumber") + public String getContactNumber() { + return contactNumber; + } + + @JsonProperty("contactNumber") + public void setContactNumber(String contactNumber) { + this.contactNumber = contactNumber; + } + + @JsonAnyGetter + public Map getAdditionalProperties() { + return this.additionalProperties; + } + + @JsonAnySetter + public void setAdditionalProperty(String name, Object value) { + this.additionalProperties.put(name, value); + } + + @Override + public String toString() { + return "Tenant [code=" + code + ", name=" + name + ", description=" + description + ", logoId=" + logoId + + ", imageId=" + imageId + ", domainUrl=" + domainUrl + ", type=" + type + ", twitterUrl=" + twitterUrl + + ", facebookUrl=" + facebookUrl + ", emailId=" + emailId + + ", city=" + city + ", address=" + address + ", contactNumber=" + contactNumber + + ", additionalProperties=" + additionalProperties + ", getCode()=" + getCode() + ", getName()=" + + getName() + ", getDescription()=" + getDescription() + ", getLogoId()=" + getLogoId() + + ", getImageId()=" + getImageId() + ", getDomainUrl()=" + getDomainUrl() + ", getType()=" + getType() + + ", getTwitterUrl()=" + getTwitterUrl() + ", getFacebookUrl()=" + getFacebookUrl() + ", getEmailId()=" + + getEmailId() + ", getCity()=" + getCity() + + ", getAddress()=" + getAddress() + ", getContactNumber()=" + getContactNumber() + + ", getAdditionalProperties()=" + getAdditionalProperties() + ", getClass()=" + getClass() + + ", hashCode()=" + hashCode() + ", toString()=" + super.toString() + "]"; + } + + + + +} diff --git a/dashboard-analytics/src/main/java/com/tarento/analytics/dto/Tenants.java b/dashboard-analytics/src/main/java/com/tarento/analytics/dto/Tenants.java new file mode 100644 index 000000000..a41b82e83 --- /dev/null +++ b/dashboard-analytics/src/main/java/com/tarento/analytics/dto/Tenants.java @@ -0,0 +1,66 @@ +package com.tarento.analytics.dto; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import com.fasterxml.jackson.annotation.JsonAnyGetter; +import com.fasterxml.jackson.annotation.JsonAnySetter; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonPropertyOrder; + +@JsonInclude(JsonInclude.Include.NON_NULL) +@JsonPropertyOrder({ "tenantId", "moduleName", "tenants" }) +public class Tenants { + + @JsonProperty("tenantId") + private String tenantId; + @JsonProperty("moduleName") + private String moduleName; + @JsonProperty("tenants") + private List tenants = null; + @JsonIgnore + private Map additionalProperties = new HashMap(); + + @JsonProperty("tenantId") + public String getTenantId() { + return tenantId; + } + + @JsonProperty("tenantId") + public void setTenantId(String tenantId) { + this.tenantId = tenantId; + } + + @JsonProperty("moduleName") + public String getModuleName() { + return moduleName; + } + + @JsonProperty("moduleName") + public void setModuleName(String moduleName) { + this.moduleName = moduleName; + } + + @JsonProperty("tenants") + public List getTenants() { + return tenants; + } + + @JsonProperty("tenants") + public void setTenants(List tenants) { + this.tenants = tenants; + } + + @JsonAnyGetter + public Map getAdditionalProperties() { + return this.additionalProperties; + } + + @JsonAnySetter + public void setAdditionalProperty(String name, Object value) { + this.additionalProperties.put(name, value); + } + +} \ No newline at end of file diff --git a/dashboard-analytics/src/main/java/com/tarento/analytics/dto/UserDto.java b/dashboard-analytics/src/main/java/com/tarento/analytics/dto/UserDto.java new file mode 100644 index 000000000..e548e8b4c --- /dev/null +++ b/dashboard-analytics/src/main/java/com/tarento/analytics/dto/UserDto.java @@ -0,0 +1,69 @@ +package com.tarento.analytics.dto; + +import java.util.List; + +public class UserDto { + + private long id; + + private String userName; + + private String emailId; + + private List roles; + + private String orgId; + + private String countryCode; + + public long getId() { + return id; + } + + public void setId(long id) { + this.id = id; + } + + public String getUserName() { + return userName; + } + + public void setUserName(String userName) { + this.userName = userName; + } + + public String getEmailId() { + return emailId; + } + + public void setEmailId(String emailId) { + this.emailId = emailId; + } + + public List getRoles() { + return roles; + } + + public void setRoles(List roles) { + this.roles = roles; + } + + public String getOrgId() { + return orgId; + } + + public void setOrgId(String orgId) { + this.orgId = orgId; + } + + public String getCountryCode() { + return countryCode; + } + + public void setCountryCode(String countryCode) { + this.countryCode = countryCode; + } + + + +} diff --git a/dashboard-analytics/src/main/java/com/tarento/analytics/dto/Visualization.java b/dashboard-analytics/src/main/java/com/tarento/analytics/dto/Visualization.java new file mode 100644 index 000000000..41a9cd654 --- /dev/null +++ b/dashboard-analytics/src/main/java/com/tarento/analytics/dto/Visualization.java @@ -0,0 +1,22 @@ +package com.tarento.analytics.dto; + +public class Visualization { + + private String type; + private String code; + public String getType() { + return type; + } + public void setType(String type) { + this.type = type; + } + public String getCode() { + return code; + } + public void setCode(String code) { + this.code = code; + } + + + +} diff --git a/dashboard-analytics/src/main/java/com/tarento/analytics/enums/ChartType.java b/dashboard-analytics/src/main/java/com/tarento/analytics/enums/ChartType.java new file mode 100644 index 000000000..4e1377e63 --- /dev/null +++ b/dashboard-analytics/src/main/java/com/tarento/analytics/enums/ChartType.java @@ -0,0 +1,31 @@ +package com.tarento.analytics.enums; + +import org.apache.commons.lang3.StringUtils; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonValue; + +public enum ChartType { + + XTABLE("xtable"), TABLE("table"), PERFORM("perform"), METRIC("metric"), PIE("pie"), LINE("line"); + + private String value; + + ChartType(final String value) { + this.value = value; + } + + + @JsonValue + public String toString() { + return value; + } + + @JsonCreator + public static ChartType fromValue(final String passedValue) { + for (final ChartType obj : ChartType.values()) + if (String.valueOf(obj.value).toLowerCase().equals(passedValue.toLowerCase())) + return obj; + return null; + } +} diff --git a/dashboard-analytics/src/main/java/com/tarento/analytics/exception/AINException.java b/dashboard-analytics/src/main/java/com/tarento/analytics/exception/AINException.java new file mode 100644 index 000000000..6e86721c3 --- /dev/null +++ b/dashboard-analytics/src/main/java/com/tarento/analytics/exception/AINException.java @@ -0,0 +1,29 @@ +package com.tarento.analytics.exception; + +public class AINException extends Exception{ + + + //TODO- wrapped up with other exceptions. + private static final long serialVersionUID = 1L; + + String errorCode; + + String errorMessage; + + public AINException(String errorCode, String errorMessage) { + this.errorCode= errorCode; + this.errorMessage = errorMessage; + } + + public String getErrorCode() { + return errorCode; + } + + + public String getErrorMessage() { + return errorMessage; + } + + + +} diff --git a/dashboard-analytics/src/main/java/com/tarento/analytics/handler/AdvanceTableChartResponseHandler.java b/dashboard-analytics/src/main/java/com/tarento/analytics/handler/AdvanceTableChartResponseHandler.java new file mode 100644 index 000000000..143c742ae --- /dev/null +++ b/dashboard-analytics/src/main/java/com/tarento/analytics/handler/AdvanceTableChartResponseHandler.java @@ -0,0 +1,212 @@ +package com.tarento.analytics.handler; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.JsonNodeFactory; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.tarento.analytics.dto.AggregateDto; +import com.tarento.analytics.dto.AggregateRequestDto; +import com.tarento.analytics.dto.Data; +import com.tarento.analytics.dto.Plot; +import com.tarento.analytics.helper.ComputedFieldFactory; +import com.tarento.analytics.helper.IComputedField; +import com.tarento.analytics.model.ComputedFields; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +import java.io.IOException; +import java.util.*; +import java.util.stream.Collectors; + +/** + * This handles ES response for single index, multiple index to represent data as pie figure + * Creates plots by merging/computing(by summation) index values for same key + * + */ +@Component +public class AdvanceTableChartResponseHandler implements IResponseHandler { + public static final Logger logger = LoggerFactory.getLogger(AdvanceTableChartResponseHandler.class); + + @Autowired + private ObjectMapper mapper; + + @Autowired + private ComputedFieldFactory computedFieldFactory; + + + @Override + public AggregateDto translate(AggregateRequestDto requestDto, ObjectNode aggregations) throws IOException { + + JsonNode aggregationNode = aggregations.get(AGGREGATIONS); + JsonNode chartNode = requestDto.getChartNode(); + String plotLabel = chartNode.get(PLOT_LABEL).asText(); + JsonNode computedFields = chartNode.get(COMPUTED_FIELDS); + JsonNode excludedFields = chartNode.get(EXCLUDED_COLUMNS); + + boolean executeComputedFields = computedFields !=null && computedFields.isArray(); + List aggrNodes = aggregationNode.findValues(BUCKETS); + boolean isPathSpecified = chartNode.get(IResponseHandler.AGGS_PATH)!=null && chartNode.get(IResponseHandler.AGGS_PATH).isArray(); + ArrayNode aggrsPaths = isPathSpecified ? (ArrayNode) chartNode.get(IResponseHandler.AGGS_PATH) : JsonNodeFactory.instance.arrayNode(); + + + + int[] idx = { 1 }; + List dataList = new ArrayList<>(); + Map> mappings = new HashMap<>(); + + + aggrNodes.stream().forEach(node -> { + + ArrayNode buckets = (ArrayNode) node; + buckets.forEach(bucket -> { + + Map plotMap = new LinkedHashMap<>(); + String key = bucket.get(IResponseHandler.KEY).asText(); + + //If aggrPath is specified. + if(aggrsPaths.size()>0){ + processWithSpecifiedKeys(aggrsPaths, bucket, mappings, key, plotMap); + + } else { + processNestedObjects(bucket, mappings, key, plotMap); + } + + if (plotMap.size() > 0) { + Map plots = new LinkedHashMap<>(); + Plot sno = new Plot(SERIAL_NUMBER, null, TABLE_TEXT); + sno.setLabel("" + idx[0]++); + Plot plotkey = new Plot(plotLabel.isEmpty() ? TABLE_KEY : plotLabel, null, TABLE_TEXT); + plotkey.setLabel(key); + + plots.put(SERIAL_NUMBER, sno); + plots.put(plotLabel.isEmpty() ? TABLE_KEY : plotLabel, plotkey); + plots.putAll(plotMap); + mappings.put(key, plots); + + } + + }); + + }); + mappings.entrySet().stream().parallel().forEach(plotMap -> { + List plotList = plotMap.getValue().values().stream().parallel().collect(Collectors.toList()); + //filter out data object with all zero data. + List filterPlot = plotList.stream().filter(c -> (!c.getName().equalsIgnoreCase(SERIAL_NUMBER) && !c.getName().equalsIgnoreCase(plotLabel) && c.getValue() != 0.0)).collect(Collectors.toList()); + + if(filterPlot.size()>0){ + Data data = new Data(plotMap.getKey(), Integer.parseInt(String.valueOf(plotMap.getValue().get(SERIAL_NUMBER).getLabel())), null); + data.setPlots(plotList); + // + if(executeComputedFields){ + try { + + List computedFieldsList = mapper.readValue(computedFields.toString(), new TypeReference>(){}); + computedFieldsList.forEach(cfs -> { + IComputedField computedFieldObject = computedFieldFactory.getInstance(cfs.getActionName()); + computedFieldObject.set(requestDto, cfs.getPostAggregationTheory()); + computedFieldObject.add(data, cfs.getFields(), cfs.getNewField() ); + + }); + // exclude the fields no to be displayed + if(excludedFields!=null){ + List list = mapper.readValue(excludedFields.toString(), new TypeReference>(){}); + List removeplots = data.getPlots().stream().filter(c -> list.contains(c.getName())).collect(Collectors.toList()); + data.getPlots().removeAll(removeplots); + } + + + } catch (Exception e){ + logger.error("execution of computed field :"+e.getMessage()); + } + } + dataList.add(data); + } + + }); + //dataList.sort((o1, o2) -> ((Integer) o1.getHeaderValue()).compareTo((Integer) o2.getHeaderValue())); + return getAggregatedDto(chartNode, dataList, requestDto.getVisualizationCode()); + + } + + + /** + * Preparing the plots + * @param bucketNode + * @param mappings + * @param key + * @param headerName + * @param plotMap + */ + private void process(JsonNode bucketNode, Map> mappings, String key, String headerName, Map plotMap){ + JsonNode valNode = bucketNode.findValue(VALUE) != null ? bucketNode.findValue(VALUE) : bucketNode.findValue(DOC_COUNT); + Double value = valNode.isDouble() ? valNode.asDouble() : valNode.asInt(); + String dataType = valNode.isDouble() ? "amount" : "number"; // to move to config or constants + //String headerName = bucketNode.findValue(KEY).asText(); + Plot plot = new Plot(headerName, value, dataType); + + if (mappings.containsKey(key)) { + double newval = mappings.get(key).get(headerName) == null ? value : (mappings.get(key).get(headerName).getValue() + value); + plot.setValue(newval); + mappings.get(key).put(headerName, plot); + } else { + plotMap.put(headerName, plot); + } + } + + /** + * Recursively processing the nodes + * @param node + * @param mappings + * @param key + * @param plotMap + */ + private void processNestedObjects(JsonNode node, Map> mappings, String key, Map plotMap ){ + + Iterator fieldNames = node.fieldNames(); + while(fieldNames.hasNext()) { + String fieldName = fieldNames.next(); + if(node.get(fieldName).isArray()){ + ArrayNode bucketNodes = (ArrayNode) node.get(fieldName); + bucketNodes.forEach(bucketNode -> { + process(bucketNode, mappings, key, bucketNode.findValue(KEY).asText() , plotMap); + }); + + } else if(node.get(fieldName).isObject() && node.get(fieldName).get(VALUE)!=null){ + process(node.get(fieldName), mappings, key, fieldName , plotMap); + + } else { + processNestedObjects(node.get(fieldName), mappings, key, plotMap ); + } + + } + + + } + + private void processWithSpecifiedKeys(ArrayNode aggrsPaths, JsonNode bucket, Map> mappings, String key, Map plotMap ){ + + aggrsPaths.forEach(headerPath -> { + JsonNode valueNode = bucket.findValue(headerPath.asText()); + //Double value = (null == valueNode || null == valueNode.get(VALUE)) ? 0.0 : valueNode.get(VALUE).asDouble(); + Double doc_value = 0.0; + if(valueNode!=null) + doc_value = (null == valueNode.findValue(DOC_COUNT)) ? 0.0 : valueNode.findValue(DOC_COUNT).asDouble(); + Double value = (null == valueNode || null == valueNode.findValue(VALUE)) ? doc_value : valueNode.findValue(VALUE).asDouble(); + String dataType = valueNode.findValue(VALUE)!=null? (valueNode.findValue(VALUE).isDouble() ? "amount" : "number") : "number" ; + + Plot plot = new Plot(headerPath.asText(), value, dataType); + if (mappings.containsKey(key)) { + double newval = mappings.get(key).get(headerPath.asText()) == null ? value : (mappings.get(key).get(headerPath.asText()).getValue() + value); + plot.setValue(newval); + mappings.get(key).put(headerPath.asText(), plot); + } else { + plotMap.put(headerPath.asText(), plot); + } + }); + } + +} diff --git a/dashboard-analytics/src/main/java/com/tarento/analytics/handler/IPostResponseHandler.java b/dashboard-analytics/src/main/java/com/tarento/analytics/handler/IPostResponseHandler.java new file mode 100644 index 000000000..57ceb2bf4 --- /dev/null +++ b/dashboard-analytics/src/main/java/com/tarento/analytics/handler/IPostResponseHandler.java @@ -0,0 +1,15 @@ +package com.tarento.analytics.handler; + +import com.fasterxml.jackson.databind.node.ObjectNode; + +public interface IPostResponseHandler { + + + /** + * Intecepts the response tree + * @param responseNode + */ + public void postResponse(ObjectNode responseNode); + + +} diff --git a/dashboard-analytics/src/main/java/com/tarento/analytics/handler/IResponseHandler.java b/dashboard-analytics/src/main/java/com/tarento/analytics/handler/IResponseHandler.java new file mode 100644 index 000000000..fcd0b1578 --- /dev/null +++ b/dashboard-analytics/src/main/java/com/tarento/analytics/handler/IResponseHandler.java @@ -0,0 +1,196 @@ +package com.tarento.analytics.handler; + +import java.io.IOException; +import java.util.*; +import java.util.function.Function; +import java.util.stream.Collectors; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; + +import com.tarento.analytics.dto.AggregateDto; +import com.tarento.analytics.dto.AggregateRequestDto; +import com.tarento.analytics.dto.Data; +import com.tarento.analytics.dto.Plot; +import com.tarento.analytics.enums.ChartType; +import org.apache.commons.collections.CollectionUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Handles Elastic search consolidate responses + */ +public interface IResponseHandler { + + public static final Logger logger = LoggerFactory.getLogger(IResponseHandler.class); + + public static final String API_CONFIG_JSON = "ChartApiConfig.json"; + public static final String AGGS_PATH = "aggregationPaths"; + + public static final String CHART_NAME = "chartName"; + public static final String CHART_TYPE = "chartType"; + public static final String DRILL_CHART = "drillChart"; + public static final String VALUE_TYPE = "valueType"; + public static final String FILTER_KEYS = "filterKeys"; + + // Table Chart Keys + public static final String SERIAL_NUMBER = "S.N."; + public static final String TABLE_TEXT = "text" ; + public static final String TABLE_KEY = "Key"; + + + // TODO remove the specific column names. + public static final String TOTAL_COLLECTION = "Total Collection"; + public static final String TARGET_COLLECTION = "Target Collection"; + public static final String TARGET_ACHIEVED = "Target Achievement"; + + public static final String PT_DDR_BOUNDARY = "demandCollectionIndexDDRRevenue"; + public static final String PT_BOUNDARY = "demandCollectionIndexBoundaryRevenue"; + public static final String PT_BOUNDARY_DRILL = "boundaryDrillDown"; + public static final String TL_DDR_BOUNDARY = "licenseIssuedDDRRevenue"; + public static final String TL_BOUNDARY = "licenseIssuedBoundaryRevenue"; + public static final String TL_BOUNDARY_DRILL = "licenseIssuedBoundaryDrillDown"; + + + + public final String ASC = "asc"; + public final String DESC = "desc"; + public final String RANK = "Rank"; + public final String AGGREGATIONS = "aggregations"; + public final String PLOT_LABEL = "plotLabel"; + public final String COMPUTED_FIELDS = "computedFields"; + public final String EXCLUDED_COLUMNS = "excludedColumns"; + public final String LIMIT = "limit"; + public final String ORDER = "order"; + public final String ACTION = "action"; + public final String TYPE_MAPPING = "pathDataTypeMapping"; + + public static String BUCKETS = "buckets"; + public static String KEY = "key"; + public static String VALUE = "value"; + + public final String PERCENTAGE = "percentage"; + public final String DOC_COUNT = "doc_count"; + + public static final String POST_AGGREGATION_THEORY = "postAggregationTheory"; + + public static Double BOUNDARY_VALUE = 50.0; + + /** + * Translate the consolidated/aggregated response + * + * @param requestDto + * @param aggregations + * @return + * @throws IOException + */ + public AggregateDto translate(AggregateRequestDto requestDto, ObjectNode aggregations) throws IOException; + + /** + * Prepare aggregated dato for a chart node + * + * @param chartNode + * @param dataList - data plots object + * @return + */ + default AggregateDto getAggregatedDto(JsonNode chartNode, List dataList, String visualizationCode) { + AggregateDto aggregateDto = new AggregateDto(); + aggregateDto.setVisualizationCode(visualizationCode); + aggregateDto.setDrillDownChartId(chartNode.get(DRILL_CHART).asText()); + ChartType chartType = ChartType.fromValue(chartNode.get(CHART_TYPE).asText()); + aggregateDto.setChartType(chartType); + aggregateDto.setData(dataList); + if(null!=chartNode.get(FILTER_KEYS)) + aggregateDto.setFilter((ArrayNode) chartNode.get(FILTER_KEYS)); + return aggregateDto; + } + + /** + * Append computed field for a given Data, for its existing fields + * computes as partfield/wholefield * 100 + * + * @param data + * @param newfield + * @param partField + * @param wholeField + */ + default void addComputedField(Data data, String newfield, String partField, String wholeField) { + try { + Map plotMap = data.getPlots().stream().parallel().collect(Collectors.toMap(Plot::getName, Function.identity())); + + if (plotMap.get(partField).getValue() == 0.0 || plotMap.get(wholeField).getValue() == 0.0) { + data.getPlots().add(new Plot(newfield, 0.0, "percentage")); + } else { + double fieldValue = plotMap.get(partField).getValue() / plotMap.get(wholeField).getValue() * 100; + data.getPlots().add(new Plot(newfield, fieldValue, "percentage")); + + } + } catch (Exception e) { + data.getPlots().add(new Plot(newfield, 0.0, "percentage")); + } + + } + + /** + * Computes the percentage from 0th and 1st index of list + * Ex: 0th element/1st element * 100 + * @param values + * @return + */ + default Double percentageValue(List values) { + double val = (values.get(0)/values.get(1) * 100); + return (values.size() > 1 && values.get(0) != 0.0 && values.get(1) != 0.0) ? val : 0.0; + } + + + /** + * Computes the percentage from 1st & 2nd element of collection + * Ex: first element/second element * 100 + * @param values + * @return + */ + default Double getPercentage(Map values, String partField, String wholeField) { + + double val = (values.get(partField)/ values.get(wholeField) * 100); + return (values.size() > 1 && values.get(partField) != 0.0 && values.get(wholeField) != 0.0) ? val: 0.0; + } + + /** + * Adding missing plot elements with cumulative data + * @param plotKeys - all required plot key + * @param data + * @param symbol + */ + default void appendMissingPlot(Set plotKeys, Data data, String symbol, boolean isCumulative) { + + //To maintain the sorted plots list order + Map sortedMap = data.getPlots().stream() + .collect(Collectors.toMap( + Plot::getName, + plot -> plot, + (u,v) -> { throw new IllegalStateException(String.format("Duplicate key %s", u)); }, + LinkedHashMap::new + )); + + logger.info(data.getHeaderName() + " existing keys: "+sortedMap.keySet()+ "& size:"+sortedMap.keySet().size()); + + Collection allKeysMinusDataKeys = CollectionUtils.subtract(plotKeys, sortedMap.keySet()); + logger.info(data.getHeaderName() +" missing keys: "+allKeysMinusDataKeys); + + + for(String plKey:allKeysMinusDataKeys){ + sortedMap.put(plKey, new Plot(plKey, new Double("0"), symbol)); + if(isCumulative){ + List keys = sortedMap.keySet().stream().collect(Collectors.toList()); + int index = keys.indexOf(plKey); + double value = index>0 ? sortedMap.get(keys.get(index-1)).getValue():0.0; + sortedMap.get(plKey).setValue(value); + } + } + logger.info("after appending missing plots : "+ sortedMap); + data.setPlots(sortedMap.values().stream().collect(Collectors.toList())); + } + +} diff --git a/dashboard-analytics/src/main/java/com/tarento/analytics/handler/InsightsHandler.java b/dashboard-analytics/src/main/java/com/tarento/analytics/handler/InsightsHandler.java new file mode 100644 index 000000000..dee91a02c --- /dev/null +++ b/dashboard-analytics/src/main/java/com/tarento/analytics/handler/InsightsHandler.java @@ -0,0 +1,41 @@ +package com.tarento.analytics.handler; + +import java.util.List; + +import com.tarento.analytics.dto.AggregateDto; +import com.tarento.analytics.dto.Data; +import com.tarento.analytics.enums.ChartType; +import com.tarento.analytics.model.InsightsConfiguration; + +public interface InsightsHandler { + + public static final String CHART_NAME = "chartName"; + public static final String CHART_TYPE = "chartType"; + public static final String DRILL_CHART = "drillChart"; + public static final String VALUE_TYPE = "valueType"; + public static final String FILTER_KEYS = "filterKeys"; + public static final String INSIGHT_WIDGET_NAME = "INSIGHTS"; + public static final String INDICATOR_PLACEHOLDER = "$indicator"; + public static final String VALUE_PLACEHOLDER = "$value"; + public static final String INSIGHT_INTERVAL_PLACEHOLDER = "$insightInterval"; + public static final String INSIGHT_INDICATOR_POSITIVE = "upper_green"; + public static final String INSIGHT_INDICATOR_NEGATIVE = "lower_red"; + public static final String POSITIVE = "+"; + public static final String NEGATIVE = "-"; + + public static final String INSIGHT_NUMBER_DIFFERENCE = "differenceOfNumbers" ; + public static final String INSIGHT_PERCENTAGE_DIFFERENCE = "differenceOfPercentage" ; + + AggregateDto getInsights(AggregateDto aggregateDto, String visualizationCode, String moduleLevel, InsightsConfiguration insightsConfig); + + default AggregateDto getAggregatedDto(List dataList, String visualizationCode) { + AggregateDto aggregateDto = new AggregateDto(); + aggregateDto.setVisualizationCode(visualizationCode); + aggregateDto.setDrillDownChartId("none"); + ChartType chartType = ChartType.fromValue("metric"); + aggregateDto.setChartType(chartType); + aggregateDto.setData(dataList); + return aggregateDto; + } + +} diff --git a/dashboard-analytics/src/main/java/com/tarento/analytics/handler/InsightsHandlerFactory.java b/dashboard-analytics/src/main/java/com/tarento/analytics/handler/InsightsHandlerFactory.java new file mode 100644 index 000000000..7167cfcc7 --- /dev/null +++ b/dashboard-analytics/src/main/java/com/tarento/analytics/handler/InsightsHandlerFactory.java @@ -0,0 +1,19 @@ +package com.tarento.analytics.handler; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +import com.tarento.analytics.enums.ChartType; + +@Component +public class InsightsHandlerFactory { + + @Autowired + private MetricsInsightsHandler metricInsightsHandler; + + public InsightsHandler getInstance(ChartType chartType) { + if(chartType == ChartType.METRIC) + return metricInsightsHandler; + return metricInsightsHandler; + } +} diff --git a/dashboard-analytics/src/main/java/com/tarento/analytics/handler/LineChartResponseHandler.java b/dashboard-analytics/src/main/java/com/tarento/analytics/handler/LineChartResponseHandler.java new file mode 100644 index 000000000..22ea1888f --- /dev/null +++ b/dashboard-analytics/src/main/java/com/tarento/analytics/handler/LineChartResponseHandler.java @@ -0,0 +1,204 @@ +package com.tarento.analytics.handler; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Calendar; +import java.util.Date; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.tarento.analytics.helper.ComputedFieldFactory; +import com.tarento.analytics.helper.IComputedField; +import com.tarento.analytics.model.ComputedFields; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.tarento.analytics.constant.Constants; +import com.tarento.analytics.dto.AggregateDto; +import com.tarento.analytics.dto.AggregateRequestDto; +import com.tarento.analytics.dto.Data; +import com.tarento.analytics.dto.Plot; + +import static com.tarento.analytics.constant.Constants.JsonPaths.DAYS; + +/** + * This handles ES response for single index, multiple index to represent data as line chart + * Creates plots by merging/computing(by summation) index values for same key + * AGGS_PATH : this defines the path/key to be used to search the tree + * VALUE_TYPE : defines the data type for the value formed, this could be amount, percentage, number + * + */ +@Component +public class LineChartResponseHandler implements IResponseHandler { + public static final Logger logger = LoggerFactory.getLogger(LineChartResponseHandler.class); + @Autowired + private ObjectMapper mapper; + @Autowired + private ComputedFieldFactory computedFieldFactory; + + @Override + public AggregateDto translate(AggregateRequestDto requestDto, ObjectNode aggregations) throws IOException { + + List dataList = new LinkedList<>(); + + //String json = "{\"ptindex-v1\":{\"Closed Application\":{\"buckets\":[{\"key_as_string\":\"2018-11-12T00:00:00.000Z\",\"key\":1541980800000,\"doc_count\":1,\"Applications Closed\":{\"buckets\":{\"closed\":{\"doc_count\":0,\"Count\":{\"value\":0}}}}}]},\"Total Application\":{\"buckets\":[{\"key_as_string\":\"2018-11-12T00:00:00.000Z\",\"key\":1541980800000,\"doc_count\":1,\"Count\":{\"value\":1}}]}},\"tlindex-v1\":{\"Closed Application\":{\"buckets\":[{\"key_as_string\":\"2019-04-29T00:00:00.000Z\",\"key\":1556496000000,\"doc_count\":6,\"Applications Closed\":{\"buckets\":{\"closed\":{\"doc_count\":0,\"Count\":{\"value\":0}},\"resolved\":{\"doc_count\":0,\"Count\":{\"value\":0}}}}}]},\"Total Application\":{\"buckets\":[{\"key\":1555891200000,\"doc_count\":1,\"Count\":{\"value\":1}},{\"key\":1556496000000,\"doc_count\":0,\"Count\":{\"value\":0}}]}},\"pgrindex-v1\":{\"Closed Application\":{\"buckets\":[{\"key\":1564963200000,\"doc_count\":438,\"Applications Closed\":{\"buckets\":{\"closed\":{\"doc_count\":5,\"Count\":{\"value\":5}}}}}]},\"Total Application\":{\"buckets\":[{\"key\":1564963200000,\"doc_count\":438,\"Count\":{\"value\":438}},{\"key\":1574035200000,\"doc_count\":3,\"Count\":{\"value\":3}}]}}}"; + JsonNode aggregationNode = aggregations.get(AGGREGATIONS); + JsonNode chartNode = requestDto.getChartNode(); + boolean isRequestInterval = null == requestDto.getRequestDate() ? false : requestDto.getRequestDate().getInterval()!=null && !requestDto.getRequestDate().getInterval().isEmpty(); + String interval = isRequestInterval ? requestDto.getRequestDate().getInterval(): chartNode.get(Constants.JsonPaths.INTERVAL).asText(); + if(interval == null || interval.isEmpty()){ + throw new RuntimeException("Interval must have value from config or request"); + } + + String symbol = chartNode.get(IResponseHandler.VALUE_TYPE).asText(); + ArrayNode aggrsPaths = (ArrayNode) chartNode.get(IResponseHandler.AGGS_PATH); + Set plotKeys = new LinkedHashSet<>(); + boolean isCumulative = chartNode.get("isCumulative").asBoolean(); + + JsonNode computedFields = chartNode.get(COMPUTED_FIELDS); + boolean executeComputedFields = computedFields !=null && computedFields.isArray(); + + + aggrsPaths.forEach(headerPath -> { + List aggrNodes = aggregationNode.findValues(headerPath.asText()); + + Map plotMap = new LinkedHashMap<>(); + List totalValues = new ArrayList<>(); + aggrNodes.stream().forEach(aggrNode -> { + if (aggrNode.findValues(IResponseHandler.BUCKETS).size() > 0) { + + ArrayNode buckets = (ArrayNode) aggrNode.findValues(IResponseHandler.BUCKETS).get(0); + buckets.forEach(bucket -> { + String bkey = bucket.findValue(IResponseHandler.KEY).asText(); + String key = getIntervalKey(bkey, Constants.Interval.valueOf(interval)); + + plotKeys.add(key); + double previousVal = !isCumulative ? 0.0 : (totalValues.size()>0 ? totalValues.get(totalValues.size()-1):0.0); + + double value = 0.0; + if(executeComputedFields){ + try { + + List computedFieldsList = mapper.readValue(computedFields.toString(), new TypeReference>(){}); + + for(ComputedFields cfs :computedFieldsList){ + IComputedField computedFieldObject = computedFieldFactory.getInstance(cfs.getActionName()); + computedFieldObject.set(requestDto, cfs.getPostAggregationTheory()); + computedFieldObject.add(bucket, cfs.getFields(), cfs.getNewField()); + + if(symbol.equals(DAYS)){ + + long milidiff = bucket.findValue(cfs.getNewField()).get(IResponseHandler.VALUE).asLong(); + long days = TimeUnit.MILLISECONDS.toDays(milidiff); + value = previousVal + (days); + + } else { + value = previousVal + (bucket.findValue(cfs.getNewField()).get(IResponseHandler.VALUE).asLong()); + + } + } + + } catch (Exception e){ + logger.error("execution of computed field :"+e.getMessage()); + } + + } else { + value = previousVal + ((bucket.findValue(IResponseHandler.VALUE) != null) ? bucket.findValue(IResponseHandler.VALUE).asDouble():bucket.findValue(IResponseHandler.DOC_COUNT).asDouble()); + + } + //double value = previousVal + ((bucket.findValue(IResponseHandler.VALUE) != null) ? bucket.findValue(IResponseHandler.VALUE).asDouble():bucket.findValue(IResponseHandler.DOC_COUNT).asDouble()); + + plotMap.put(key, plotMap.get(key) == null ? new Double("0") + value : plotMap.get(key) + value); + totalValues.add(value); + }); + } + }); + List plots = plotMap.entrySet().stream().map(e -> new Plot(e.getKey(), e.getValue(), symbol)).collect(Collectors.toList()); + try{ + Data data = new Data(headerPath.asText(), (totalValues==null || totalValues.isEmpty()) ? 0.0 : totalValues.stream().reduce(0.0, Double::sum), symbol); + data.setPlots(plots); + dataList.add(data); + } catch (Exception e) { + logger.error(" Legend/Header "+headerPath.asText() +" exception occurred "+e.getMessage()); + } + }); + + dataList.forEach(data -> { + appendMissingPlot(plotKeys, data, symbol, isCumulative); + }); + return getAggregatedDto(chartNode, dataList, requestDto.getVisualizationCode()); + } + + private String getIntervalKey(String epocString, Constants.Interval interval) { + try { + long epoch = Long.parseLong( epocString ); + Date expiry = new Date( epoch ); + Calendar cal = Calendar.getInstance(); + cal.setTime(expiry); + + String day = String.valueOf(cal.get(Calendar.DATE)); + String month = monthNames(cal.get(Calendar.MONTH)+1); + String year = ""+cal.get(Calendar.YEAR); + + String intervalKey = ""; + if(interval.equals(Constants.Interval.day)) { + intervalKey = day.concat("-").concat(month); + } else if(interval.equals(Constants.Interval.week)){ + intervalKey = day.concat("-").concat(month); + } else if(interval.equals(Constants.Interval.year)){ + intervalKey = year; + } else if(interval.equals(Constants.Interval.month)){ + intervalKey = month.concat("-").concat(year); + } else { + throw new RuntimeException("Invalid interval"); + } + + //String weekMonth = "Week " + cal.get(Calendar.WEEK_OF_YEAR) /*+ " : " + dayMonth*/;//+" of Month "+ (cal.get(Calendar.MONTH) + 1); + return intervalKey; + } catch (Exception e) { + return epocString; + } + } + + private String monthNames(int month) { + if(month == 1) + return "Jan"; + else if(month == 2) + return "Feb"; + else if(month == 3) + return "Mar"; + else if(month == 4) + return "Apr"; + else if(month == 5) + return "May"; + else if(month == 6) + return "Jun"; + else if(month == 7) + return "Jul"; + else if(month == 8) + return "Aug"; + else if(month == 9) + return "Sep"; + else if(month == 10) + return "Oct"; + else if(month == 11) + return "Nov"; + else if(month == 12) + return "Dec"; + else + return "Month"; + } +} diff --git a/dashboard-analytics/src/main/java/com/tarento/analytics/handler/MetricChartResponseHandler.java b/dashboard-analytics/src/main/java/com/tarento/analytics/handler/MetricChartResponseHandler.java new file mode 100644 index 000000000..d11057b14 --- /dev/null +++ b/dashboard-analytics/src/main/java/com/tarento/analytics/handler/MetricChartResponseHandler.java @@ -0,0 +1,98 @@ +package com.tarento.analytics.handler; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.tarento.analytics.ConfigurationLoader; +import com.tarento.analytics.dto.AggregateDto; +import com.tarento.analytics.dto.AggregateRequestDto; +import com.tarento.analytics.dto.Data; +import com.tarento.analytics.helper.ComputeHelper; +import com.tarento.analytics.helper.ComputeHelperFactory; + +import com.tarento.analytics.utils.ResponseRecorder; + +/** + * This handles ES response for single index, multiple index to represent single data value + * Creates plots by merging/computing(by summation or by percentage) index values for same key + * ACTION: for the chart config defines the type either summation or computing percentage + * AGGS_PATH : this defines the path/key to be used to search the tree + * + */ +@Component +public class MetricChartResponseHandler implements IResponseHandler{ + public static final Logger logger = LoggerFactory.getLogger(MetricChartResponseHandler.class); + + char insightPrefix = 'i'; + + @Autowired + ConfigurationLoader configurationLoader; + + @Autowired + ComputeHelperFactory computeHelperFactory; + + @Autowired + ResponseRecorder responseRecorder; + + @Override + public AggregateDto translate(AggregateRequestDto request, ObjectNode aggregations) throws IOException { + List dataList = new ArrayList<>(); + String requestId = request.getRequestId(); + String visualizationCode = request.getVisualizationCode(); + + JsonNode aggregationNode = aggregations.get(AGGREGATIONS); + JsonNode chartNode = null; + + if(request.getVisualizationCode().charAt(0) == insightPrefix) { + String internalChartId = request.getVisualizationCode().substring(1); + chartNode = configurationLoader.get(API_CONFIG_JSON).get(internalChartId); + } else { + chartNode = configurationLoader.get(API_CONFIG_JSON).get(request.getVisualizationCode()); + } + + + List totalValues = new ArrayList<>(); + String chartName = chartNode.get(CHART_NAME).asText(); + String action = chartNode.get(ACTION).asText(); + + List percentageList = new ArrayList<>(); + ArrayNode aggrsPaths = (ArrayNode) chartNode.get(AGGS_PATH); + + aggrsPaths.forEach(headerPath -> { + List values = aggregationNode.findValues(headerPath.asText()); + values.stream().parallel().forEach(value -> { + List valueNodes = value.findValues(VALUE).isEmpty() ? value.findValues(DOC_COUNT) : value.findValues(VALUE); + Double sum = valueNodes.stream().mapToDouble(o -> o.asDouble()).sum(); + if(action.equals(PERCENTAGE) && aggrsPaths.size()==2){ + percentageList.add(sum); + } else { + totalValues.add(sum); + } + }); + }); + + String symbol = chartNode.get(IResponseHandler.VALUE_TYPE).asText(); + try{ + Data data = new Data(chartName, action.equals(PERCENTAGE) && aggrsPaths.size()==2? percentageValue(percentageList) : (totalValues==null || totalValues.isEmpty())? 0.0 :totalValues.stream().reduce(0.0, Double::sum), symbol); + responseRecorder.put(visualizationCode, request.getModuleLevel(), data); + dataList.add(data); + if(chartNode.get(POST_AGGREGATION_THEORY) != null) { + ComputeHelper computeHelper = computeHelperFactory.getInstance(chartNode.get(POST_AGGREGATION_THEORY).asText()); + computeHelper.compute(request, dataList); + } + }catch (Exception e){ + logger.info("data chart name = "+chartName +" ex occurred "+e.getMessage()); + } + + return getAggregatedDto(chartNode, dataList, request.getVisualizationCode()); + } +} diff --git a/dashboard-analytics/src/main/java/com/tarento/analytics/handler/MetricsInsightsHandler.java b/dashboard-analytics/src/main/java/com/tarento/analytics/handler/MetricsInsightsHandler.java new file mode 100644 index 000000000..4a6594eaa --- /dev/null +++ b/dashboard-analytics/src/main/java/com/tarento/analytics/handler/MetricsInsightsHandler.java @@ -0,0 +1,60 @@ +package com.tarento.analytics.handler; + +import java.text.DecimalFormat; +import java.util.List; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +import com.tarento.analytics.dto.AggregateDto; +import com.tarento.analytics.dto.Data; +import com.tarento.analytics.dto.InsightsWidget; +import com.tarento.analytics.model.InsightsConfiguration; +import com.tarento.analytics.utils.ResponseRecorder; + +@Component +public class MetricsInsightsHandler implements InsightsHandler { + + @Autowired + private ResponseRecorder responseRecorder; + + @Override + public AggregateDto getInsights(AggregateDto aggregateDto, String visualizationCode, String moduleLevel, InsightsConfiguration insightsConfig) { + String internalVisualizationCode = visualizationCode.substring(1); + Data currentData = (Data) responseRecorder.get(internalVisualizationCode, moduleLevel); + Data pastData = (Data) responseRecorder.get(visualizationCode, moduleLevel); + String textToDisplay = insightsConfig.getTextMessage(); + String insightIndicator = ""; + if(INSIGHT_NUMBER_DIFFERENCE.equals(insightsConfig.getAction())) { + Double difference = (Double)currentData.getHeaderValue() - (Double) pastData.getHeaderValue(); + if(difference >= 0) { + Double insightValue = (difference / (Double)pastData.getHeaderValue()) * 100; + if(insightValue.isInfinite()) + return aggregateDto; + if(insightValue.isNaN()) + insightValue = 0.00; + textToDisplay = textToDisplay.replace(INDICATOR_PLACEHOLDER, POSITIVE); + textToDisplay = textToDisplay.replace(VALUE_PLACEHOLDER, String.valueOf(new DecimalFormat("#.##").format(insightValue))); + insightIndicator = INSIGHT_INDICATOR_POSITIVE; + } else { + difference = (Double) pastData.getHeaderValue() - (Double) currentData.getHeaderValue(); + Double insightValue = (difference / (Double)pastData.getHeaderValue()) * 100; + if(insightValue.isInfinite()) + return aggregateDto; + if(difference.equals(0.0) && insightValue.equals(0.0)) + return aggregateDto; + textToDisplay = textToDisplay.replace(INDICATOR_PLACEHOLDER, NEGATIVE); + textToDisplay = textToDisplay.replace(VALUE_PLACEHOLDER, String.valueOf(new DecimalFormat("#.##").format(insightValue))); + insightIndicator = INSIGHT_INDICATOR_NEGATIVE; + } + textToDisplay = textToDisplay.replace(INSIGHT_INTERVAL_PLACEHOLDER, insightsConfig.getInsightInterval()); + InsightsWidget insightsWidget = new InsightsWidget(INSIGHT_WIDGET_NAME, textToDisplay, insightIndicator, insightIndicator); + List dataList = aggregateDto.getData(); + for(Data data : dataList) { + data.setInsight(insightsWidget); + } + } + return aggregateDto; + } + +} diff --git a/dashboard-analytics/src/main/java/com/tarento/analytics/handler/PerformanceChartResponeHandler.java b/dashboard-analytics/src/main/java/com/tarento/analytics/handler/PerformanceChartResponeHandler.java new file mode 100644 index 000000000..159b1bb78 --- /dev/null +++ b/dashboard-analytics/src/main/java/com/tarento/analytics/handler/PerformanceChartResponeHandler.java @@ -0,0 +1,111 @@ +package com.tarento.analytics.handler; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.tarento.analytics.ConfigurationLoader; +import com.tarento.analytics.dto.AggregateDto; +import com.tarento.analytics.dto.AggregateRequestDto; +import com.tarento.analytics.dto.Data; +import com.tarento.analytics.dto.Plot; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +import java.io.IOException; +import java.util.*; +import java.util.stream.Collectors; + +@Component +public class PerformanceChartResponeHandler implements IResponseHandler { + + public static final Logger logger = LoggerFactory.getLogger(PerformanceChartResponeHandler.class); + + @Override + public AggregateDto translate(AggregateRequestDto requestDto, ObjectNode aggregations) throws IOException { + + JsonNode aggregationNode = aggregations.get(AGGREGATIONS); + JsonNode chartNode = requestDto.getChartNode(); + String symbol = chartNode.get(IResponseHandler.VALUE_TYPE).asText(); + String plotLabel = chartNode.get(PLOT_LABEL).asText(); + String order = chartNode.get(ORDER).asText(); + int limit = chartNode.get(LIMIT).asInt(); + + + ArrayNode aggrsPaths = (ArrayNode) chartNode.get(IResponseHandler.AGGS_PATH); + Map> mappings = new LinkedHashMap<>();//HashMap<>(); + + aggrsPaths.forEach(headerPath -> { + aggregationNode.findValues(headerPath.asText()).forEach(aggrNode -> { + if (aggrNode.findValues(IResponseHandler.BUCKETS).size() > 0) { + + ArrayNode buckets = (ArrayNode) aggrNode.findValues(IResponseHandler.BUCKETS).get(0); + buckets.forEach(bucket -> { + String key = bucket.findValue(IResponseHandler.KEY).asText(); + Double value = bucket.findValue(IResponseHandler.VALUE).asDouble(); + + if (mappings.containsKey(key)) { + Double sum = (mappings.get(key)).containsKey(headerPath.asText()) ? (mappings.get(key)).get(headerPath.asText()) + value : value; + (mappings.get(key)).put(headerPath.asText(), sum); + + } else { + Map additiveMap = new HashMap<>(); + additiveMap.put(aggrsPaths.get(0).asText(), new Double("0")); + additiveMap.put(aggrsPaths.get(1).asText(), new Double("0")); + + additiveMap.put(headerPath.asText(), value); + mappings.put(key, additiveMap); + } + }); + } + }); + }); + logger.info("performance chart data mappings : "+mappings); + List plotList = mappings.entrySet().stream().parallel().map(e -> new Plot(e.getKey(), getPercentage(e.getValue(), aggrsPaths.get(0).asText(),aggrsPaths.get(1).asText()), symbol)).collect(Collectors.toList()); + List plots = plotList.stream().filter(plot -> plot.getValue() != 0.0).collect(Collectors.toList()); + + plots.stream().parallel().forEach(item -> item.setLabel(plotLabel)); + Comparator plotValueComparator = Comparator.comparing(Plot::getValue); + plots.sort(plotValueComparator.reversed()); + return getAggregatedDto(chartNode, getDataOnPerformingOrder(plots, limit, order, symbol), requestDto.getVisualizationCode()); + } + + /** + * Prepare the plots with it's header Data in performing order + * @param plots + * @param limit n number of plots elements + * @param order top wise or bottom wise performance + * @param symbol + * @return + */ + private List getDataOnPerformingOrder(List plots, int limit, String order, String symbol) { + + List dataList = new ArrayList<>(); + if (order.equals(ASC)) { + for (int i = (plots.size() - 1); i >= 0; i--) { + dataList.add(getRankedPLot(i, symbol, plots)); + } + } else if (order.equals(DESC)) { + for (int i = 0; i < plots.size(); i++) { + dataList.add(getRankedPLot(i, symbol, plots)); + } + } + // return dataList.subList(Math.max(0, 0), Math.min(dataList.size(), limit)); + return dataList; + + } + + private Data getRankedPLot(int rank, String dataType, List plots){ + Data data = new Data(RANK, rank+1, dataType); + List p = new ArrayList(); + p.add(plots.get(rank)); + data.setPlots(p); + return data; + } + +} + + diff --git a/dashboard-analytics/src/main/java/com/tarento/analytics/handler/PieChartResponseHandler.java b/dashboard-analytics/src/main/java/com/tarento/analytics/handler/PieChartResponseHandler.java new file mode 100644 index 000000000..f79fc518e --- /dev/null +++ b/dashboard-analytics/src/main/java/com/tarento/analytics/handler/PieChartResponseHandler.java @@ -0,0 +1,76 @@ +package com.tarento.analytics.handler; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.tarento.analytics.ConfigurationLoader; +import com.tarento.analytics.dto.AggregateDto; +import com.tarento.analytics.dto.AggregateRequestDto; +import com.tarento.analytics.dto.Data; +import com.tarento.analytics.dto.Plot; +import com.tarento.analytics.enums.ChartType; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +/** + * This handles ES response for single index, multiple index to represent data as pie figure + * Creates plots by merging/computing(by summation) index values for same key + * AGGS_PATH : this defines the path/key to be used to search the tree + * VALUE_TYPE : defines the data type for the value formed, this could be amount, percentage, number + * + */ +@Component +public class PieChartResponseHandler implements IResponseHandler { + public static final Logger logger = LoggerFactory.getLogger(PieChartResponseHandler.class); + + + @Override + public AggregateDto translate(AggregateRequestDto requestDto, ObjectNode aggregations) throws IOException { + + List dataList = new ArrayList<>(); + + JsonNode aggregationNode = aggregations.get(AGGREGATIONS); + JsonNode chartNode = requestDto.getChartNode(); + String headerKey = chartNode.get(CHART_NAME).asText(); + List headerPlotList = new ArrayList<>(); + List totalValue = new ArrayList<>(); + + String symbol = chartNode.get(IResponseHandler.VALUE_TYPE).asText(); + ArrayNode aggrsPaths = (ArrayNode) chartNode.get(IResponseHandler.AGGS_PATH); + + aggrsPaths.forEach(headerPath -> { + aggregationNode.findValues(headerPath.asText()).stream().parallel().forEach(valueNode->{ + if(valueNode.has(BUCKETS)){ + JsonNode buckets = valueNode.findValue(BUCKETS); + buckets.forEach(bucket -> { + Double val = valueNode.findValues(VALUE).isEmpty() ? bucket.findValue(DOC_COUNT).asInt() : bucket.findValue(VALUE).asDouble(); + totalValue.add(val); + Plot plot = new Plot(bucket.findValue(KEY).asText(), val, symbol); + headerPlotList.add(plot); + }); + + } else { + List valueNodes = valueNode.findValues(VALUE).isEmpty() ? valueNode.findValues(DOC_COUNT) : valueNode.findValues(VALUE); + double sum = valueNodes.stream().mapToLong(o -> o.asLong()).sum(); + totalValue.add(sum); + Plot plot = new Plot(headerPath.asText(), sum, symbol); + headerPlotList.add(plot); + } + }); + }); + + Data data = new Data(headerKey, totalValue.stream().reduce(0.0, Double::sum), symbol); + data.setPlots(headerPlotList); + dataList.add(data); + + return getAggregatedDto(chartNode, dataList, requestDto.getVisualizationCode()); + + } +} diff --git a/dashboard-analytics/src/main/java/com/tarento/analytics/handler/ResponseHandlerFactory.java b/dashboard-analytics/src/main/java/com/tarento/analytics/handler/ResponseHandlerFactory.java new file mode 100644 index 000000000..65cb3aec9 --- /dev/null +++ b/dashboard-analytics/src/main/java/com/tarento/analytics/handler/ResponseHandlerFactory.java @@ -0,0 +1,60 @@ +package com.tarento.analytics.handler; + +import com.tarento.analytics.enums.ChartType; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +@Component +public class ResponseHandlerFactory { + + @Autowired + private MetricChartResponseHandler metricChartResponseHandler; + @Autowired + private LineChartResponseHandler lineChartResponseHandler; + @Autowired + private PieChartResponseHandler pieChartResponseHandler; + @Autowired + private PerformanceChartResponeHandler performingBarChartResponeHandler; + @Autowired + private TableChartResponseHandler tableChartResponseHandler; + @Autowired + private AdvanceTableChartResponseHandler advanceTableChartResponseHandler; + + + public IResponseHandler getInstance(ChartType chartType) { + + if (chartType == chartType.METRIC) { + return metricChartResponseHandler; + + } else if (chartType == chartType.LINE) { + return lineChartResponseHandler; + + } else if (chartType == chartType.PIE) { + return pieChartResponseHandler; + + } else if (chartType == chartType.PERFORM) { + return performingBarChartResponeHandler; + + } else if (chartType == chartType.TABLE) { + return tableChartResponseHandler; + + } else if (chartType == chartType.XTABLE) { + return advanceTableChartResponseHandler; + } + + return null; + } + + @Autowired + private TablePostResponseHandler tablePostResponseHandler; + public IPostResponseHandler get(ChartType chartType){ + + if(chartType == chartType.TABLE){ + return tablePostResponseHandler; + } + return null; + } + +} + + diff --git a/dashboard-analytics/src/main/java/com/tarento/analytics/handler/TableChartResponseHandler.java b/dashboard-analytics/src/main/java/com/tarento/analytics/handler/TableChartResponseHandler.java new file mode 100644 index 000000000..b7120bfb2 --- /dev/null +++ b/dashboard-analytics/src/main/java/com/tarento/analytics/handler/TableChartResponseHandler.java @@ -0,0 +1,115 @@ +package com.tarento.analytics.handler; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import com.tarento.analytics.helper.ComputedFieldHelper; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.tarento.analytics.dto.AggregateDto; +import com.tarento.analytics.dto.AggregateRequestDto; +import com.tarento.analytics.dto.Data; +import com.tarento.analytics.dto.Plot; +/** + * This handles ES response for single index, multiple index to compute performance + * Creates plots by performing ordered (ex: top n performance or last n performance) + * AGGS_PATH : configurable to this defines the path/key to be used to search the tree + * VALUE_TYPE : configurable to define the data type for the value formed, this could be amount, percentage, number + * PLOT_LABEL : configurable to define the label for the plot + * TYPE_MAPPING : defines for a plot data type + */ +@Component +public class TableChartResponseHandler implements IResponseHandler { + public static final Logger logger = LoggerFactory.getLogger(TableChartResponseHandler.class); + + @Autowired + ComputedFieldHelper computedFieldHelper; + @Override + public AggregateDto translate(AggregateRequestDto requestDto, ObjectNode aggregations) throws IOException { + + JsonNode aggregationNode = aggregations.get(AGGREGATIONS); + JsonNode chartNode = requestDto.getChartNode(); + String postAggrTheoryName = chartNode.get(POST_AGGREGATION_THEORY) == null ? "" : chartNode.get(POST_AGGREGATION_THEORY).asText(); + String plotLabel = chartNode.get(PLOT_LABEL).asText(); + ArrayNode pathDataTypeMap = (ArrayNode) chartNode.get(TYPE_MAPPING); + ArrayNode aggrsPaths = (ArrayNode) chartNode.get(IResponseHandler.AGGS_PATH); + Map> mappings = new HashMap<>(); + List aggrNodes = aggregationNode.findValues(BUCKETS); + + int[] idx = { 1 }; + + aggrNodes.stream().forEach(node -> { + ArrayNode buckets = (ArrayNode) node; + buckets.forEach(bucket -> { + Map plotMap = new LinkedHashMap<>(); + String key = bucket.findValue(IResponseHandler.KEY).asText(); + + aggrsPaths.forEach(headerPath -> { + JsonNode datatype = pathDataTypeMap.findValue(headerPath.asText()); + JsonNode valueNode = bucket.findValue(headerPath.asText()); + //Double value = (null == valueNode || null == valueNode.get(VALUE)) ? 0.0 : valueNode.get(VALUE).asDouble(); + Double doc_value = 0.0; + if(valueNode!=null) + doc_value = (null == valueNode.findValue(DOC_COUNT)) ? 0.0 : valueNode.findValue(DOC_COUNT).asDouble(); + Double value = (null == valueNode || null == valueNode.findValue(VALUE)) ? doc_value : valueNode.findValue(VALUE).asDouble(); + Plot plot = new Plot(headerPath.asText(), value, datatype.asText()); + if (mappings.containsKey(key)) { + double newval = mappings.get(key).get(headerPath.asText()) == null ? value : (mappings.get(key).get(headerPath.asText()).getValue() + value); + plot.setValue(newval); + mappings.get(key).put(headerPath.asText(), plot); + } else { + plotMap.put(headerPath.asText(), plot); + } + }); + + if (plotMap.size() > 0) { + Map plots = new LinkedHashMap<>(); + Plot sno = new Plot(SERIAL_NUMBER, null, TABLE_TEXT); + sno.setLabel("" + idx[0]++); + Plot plotkey = new Plot(plotLabel.isEmpty() ? TABLE_KEY : plotLabel, null, TABLE_TEXT); + plotkey.setLabel(key); + + plots.put(SERIAL_NUMBER, sno); + plots.put(plotLabel.isEmpty() ? TABLE_KEY : plotLabel, plotkey); + plots.putAll(plotMap); + mappings.put(key, plots); + + } + }); + + }); + + List dataList = new ArrayList<>(); + mappings.entrySet().stream().parallel().forEach(plotMap -> { + List plotList = plotMap.getValue().values().stream().parallel().collect(Collectors.toList()); + List filterPlot = plotList.stream().filter(c -> (!c.getName().equalsIgnoreCase(SERIAL_NUMBER) && !c.getName().equalsIgnoreCase(plotLabel) && c.getValue() != 0.0)).collect(Collectors.toList()); + if(filterPlot.size()>0){ + Data data = new Data(plotMap.getKey(), Integer.parseInt(String.valueOf(plotMap.getValue().get(SERIAL_NUMBER).getLabel())), null); + data.setPlots(plotList); + + if(requestDto.getVisualizationCode().equals(PT_DDR_BOUNDARY) || requestDto.getVisualizationCode().equals(PT_BOUNDARY) || requestDto.getVisualizationCode().equals(PT_BOUNDARY_DRILL) + || requestDto.getVisualizationCode().equals(TL_DDR_BOUNDARY) || requestDto.getVisualizationCode().equals(TL_BOUNDARY) || requestDto.getVisualizationCode().equals(TL_BOUNDARY_DRILL)) { + + computedFieldHelper.set(requestDto, postAggrTheoryName); + computedFieldHelper.add(data,TARGET_ACHIEVED, TOTAL_COLLECTION, TARGET_COLLECTION ); + } + dataList.add(data); + } + + }); + //dataList.sort((o1, o2) -> ((Integer) o1.getHeaderValue()).compareTo((Integer) o2.getHeaderValue())); + return getAggregatedDto(chartNode, dataList, requestDto.getVisualizationCode()); + } + +} diff --git a/dashboard-analytics/src/main/java/com/tarento/analytics/handler/TablePostResponseHandler.java b/dashboard-analytics/src/main/java/com/tarento/analytics/handler/TablePostResponseHandler.java new file mode 100644 index 000000000..65193be1c --- /dev/null +++ b/dashboard-analytics/src/main/java/com/tarento/analytics/handler/TablePostResponseHandler.java @@ -0,0 +1,69 @@ +package com.tarento.analytics.handler; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.tarento.analytics.constant.Constants; +import com.tarento.analytics.org.service.MdmsService; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +@Component +public class TablePostResponseHandler implements IPostResponseHandler { + private static Logger logger = LoggerFactory.getLogger(TablePostResponseHandler.class); + + + @Autowired + MdmsService mdmsService; + + /*String json = "{\"aggregations\":{\"dss-collection_v1\":{\"ULBs \":{\"doc_count_error_upper_bound\":0,\"sum_other_doc_count\":0,\"buckets\":[{\"key\":\"pb.derabassi\",\"doc_count\":448,\"Transactions\":{\"value\":448},\"Assessed Properties\":{\"value\":1251},\"Total Collection\":{\"value\":620938.0}},{\"key\":\"pb.nayagaon\",\"doc_count\":97,\"Transactions\":{\"value\":97},\"Assessed Properties\":{\"value\":235},\"Total Collection\":{\"value\":69108.0}}]}},\"dss-target_v1\":{\"ULBs \":{\"doc_count_error_upper_bound\":0,\"sum_other_doc_count\":0,\"buckets\":[{\"key\":\"pb.derabassi\",\"doc_count\":1,\"Target Collection\":{\"value\":1.5E7}},{\"key\":\"pb.nayagaon\",\"doc_count\":1,\"Target Collection\":{\"value\":3500000.0}}]}}}}"; + String json1 = "{\"aggregations\":{\"DDRCode \":{\"doc_count_error_upper_bound\":0,\"sum_other_doc_count\":0,\"buckets\":[{\"key\":\"1\",\"doc_count\":277,\"Transactions\":{\"value\":277},\"Assessed Properties\":{\"value\":805},\"Total Collection\":{\"value\":618472.0}},{\"key\":\"9\",\"doc_count\":2719,\"Transactions\":{\"value\":2719},\"Assessed Properties\":{\"value\":8434},\"Total Collection\":{\"value\":5572731.0}}]}}}"; + */ + + /** + * Post response handle for replacing the district names from MDMS service map + * @param responseNode + */ + @Override + public void postResponse(ObjectNode responseNode){ + try { + + List nodes = responseNode.findValues(Constants.MDMSKeys.KEY); + Set values = new HashSet(); + nodes.forEach(node -> { + if(!values.contains(node.asText())) + replaceField(responseNode, Constants.MDMSKeys.KEY, node.asText(),mdmsService.getDDRNameByCode(node.asText())); + values.add(node.asText()); + + }); + logger.info("post response handling(DDR values) "+responseNode); + + }catch (Exception e){ + logger.error("post response handling(DDR values) "+e.getMessage()); + } + + } + + + public static void replaceField(ObjectNode parent, String fieldName, String oldValue, String newValue) { + if (parent.has(fieldName) && parent.get(fieldName).asText().equalsIgnoreCase(oldValue)) { + parent.put(fieldName, newValue); + } + parent.fields().forEachRemaining(entry -> { + JsonNode entryValue = entry.getValue(); + if (entryValue.isArray()) { + for (int i = 0; i < entryValue.size(); i++) { + if (entry.getValue().get(i).isObject()) + replaceField((ObjectNode) entry.getValue().get(i), fieldName, oldValue, newValue); + } + } else if (entryValue.isObject()) { + replaceField((ObjectNode) entry.getValue(), fieldName, oldValue, newValue); + } + }); + } +} diff --git a/dashboard-analytics/src/main/java/com/tarento/analytics/helper/AdditiveComputedField.java b/dashboard-analytics/src/main/java/com/tarento/analytics/helper/AdditiveComputedField.java new file mode 100644 index 000000000..305c3af7a --- /dev/null +++ b/dashboard-analytics/src/main/java/com/tarento/analytics/helper/AdditiveComputedField.java @@ -0,0 +1,59 @@ +package com.tarento.analytics.helper; + +import com.tarento.analytics.dto.AggregateRequestDto; +import com.tarento.analytics.dto.Data; +import com.tarento.analytics.dto.Plot; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +import java.util.List; +import java.util.Map; +import java.util.function.Function; +import java.util.stream.Collectors; + +@Component +public class AdditiveComputedField implements IComputedField { + + public static final Logger logger = LoggerFactory.getLogger(AdditiveComputedField.class); + + + private String postAggrTheoryName; + private AggregateRequestDto aggregateRequestDto; + @Autowired + private ComputeHelperFactory computeHelperFactory; + + @Override + public void set(AggregateRequestDto requestDto, String postAggrTheoryName){ + this.aggregateRequestDto = requestDto; + this.postAggrTheoryName = postAggrTheoryName; + } + + @Override + public void add(Data data, List fields, String newField) { + try { + Map plotMap = data.getPlots().stream().parallel().collect(Collectors.toMap(Plot::getName, Function.identity())); + + double total = 0.0; + for (String field: fields){ + total = total+ plotMap.get(field).getValue(); + } + if(postAggrTheoryName != null && !postAggrTheoryName.isEmpty()) { + ComputeHelper computeHelper = computeHelperFactory.getInstance(postAggrTheoryName); + + total = computeHelper.compute(aggregateRequestDto,total ); + } + + + data.getPlots().add(new Plot(newField, total, "amount")); + + } catch (Exception e) { + // throw new RuntimeException("Computed field configuration not correctly provided"); + logger.error("percentage could not be computed " +e.getMessage()); + data.getPlots().add(new Plot(newField, 0.0, "amount")); + } + + } +} + diff --git a/dashboard-analytics/src/main/java/com/tarento/analytics/helper/AverageComputedField.java b/dashboard-analytics/src/main/java/com/tarento/analytics/helper/AverageComputedField.java new file mode 100644 index 000000000..361ec32e5 --- /dev/null +++ b/dashboard-analytics/src/main/java/com/tarento/analytics/helper/AverageComputedField.java @@ -0,0 +1,63 @@ +package com.tarento.analytics.helper; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.JsonNodeFactory; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.tarento.analytics.dto.AggregateRequestDto; +import com.tarento.analytics.handler.IResponseHandler; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +import java.util.ArrayList; +import java.util.List; + +@Component +public class AverageComputedField implements IComputedField { + + public static final Logger logger = LoggerFactory.getLogger(AverageComputedField.class); + + + private String postAggrTheoryName; + private AggregateRequestDto aggregateRequestDto; + @Autowired + private ComputeHelperFactory computeHelperFactory; + + @Override + public void set(AggregateRequestDto requestDto, String postAggrTheoryName){ + this.aggregateRequestDto = requestDto; + this.postAggrTheoryName = postAggrTheoryName; + } + + @Override + public void add(ObjectNode data, List fields, String newField) { + ObjectNode averageNode = JsonNodeFactory.instance.objectNode(); + + try { + + List values = data.findValues(fields.get(0)); + List vals = new ArrayList<>(); + for(JsonNode valueNode : values){ + vals.add(valueNode.get(IResponseHandler.VALUE).asLong()); + } + + Long sum = vals.stream().reduce(new Long(0), (a, b) -> a + b); + logger.info("sum value:: "+sum+" vals.size():: "+vals.size()); + Long average = sum >= vals.size()? sum/vals.size() : 0; + logger.info("average value:: "+average+" total value:: "+sum); + + averageNode.put(IResponseHandler.VALUE, average); + data.set(newField, averageNode); + + + } catch (Exception e) { + // throw new RuntimeException("Computed field configuration not correctly provided"); + logger.error("average could not be computed " +e.getMessage()); + averageNode.put(IResponseHandler.VALUE,0); + data.set(newField, averageNode); + } + + } +} + diff --git a/dashboard-analytics/src/main/java/com/tarento/analytics/helper/ComputeHelper.java b/dashboard-analytics/src/main/java/com/tarento/analytics/helper/ComputeHelper.java new file mode 100644 index 000000000..3c50469fe --- /dev/null +++ b/dashboard-analytics/src/main/java/com/tarento/analytics/helper/ComputeHelper.java @@ -0,0 +1,19 @@ +package com.tarento.analytics.helper; + +import java.util.List; + +import com.tarento.analytics.dto.AggregateRequestDto; +import com.tarento.analytics.dto.Data; + +/** + * Compute Helper Inferface which receives the Request and the List of Data + * Implementations will derive as to what has to be the computation based on the Business Logic Specifications + * @author darshan + * + */ +public interface ComputeHelper { + + public List compute(AggregateRequestDto request, List data); + public Double compute(AggregateRequestDto request, double value); + +} diff --git a/dashboard-analytics/src/main/java/com/tarento/analytics/helper/ComputeHelperFactory.java b/dashboard-analytics/src/main/java/com/tarento/analytics/helper/ComputeHelperFactory.java new file mode 100644 index 000000000..f6f6a555f --- /dev/null +++ b/dashboard-analytics/src/main/java/com/tarento/analytics/helper/ComputeHelperFactory.java @@ -0,0 +1,28 @@ +package com.tarento.analytics.helper; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +import com.tarento.analytics.constant.Constants; + +/** + * Factory Class to supply the right implementation for the Post Aggregation Computation Business Logic + * which has to be supplied based on the Configuration written in the Chart API Configuration + * @author darshan + * + */ +@Component +public class ComputeHelperFactory { + + @Autowired + private TargetPerDateComputeHelper targetPerDateComputeHelper; + + public ComputeHelper getInstance(String intent) { + + if (intent.equals(Constants.PostAggregationTheories.RESPONSE_DIFF_DATES)) { + return targetPerDateComputeHelper; + } + return null; + } + +} diff --git a/dashboard-analytics/src/main/java/com/tarento/analytics/helper/ComputedFieldFactory.java b/dashboard-analytics/src/main/java/com/tarento/analytics/helper/ComputedFieldFactory.java new file mode 100644 index 000000000..3af853d7c --- /dev/null +++ b/dashboard-analytics/src/main/java/com/tarento/analytics/helper/ComputedFieldFactory.java @@ -0,0 +1,33 @@ +package com.tarento.analytics.helper; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +@Component +public class ComputedFieldFactory { + + @Autowired + private PercentageComputedField percentageComputedField; + @Autowired + private AverageComputedField averageComputedField; + @Autowired + private AdditiveComputedField additiveComputedField; + + public IComputedField getInstance(String className){ + + if(className.equalsIgnoreCase(percentageComputedField.getClass().getSimpleName())){ + return percentageComputedField; + + } else if(className.equalsIgnoreCase(averageComputedField.getClass().getSimpleName())) { + return averageComputedField; + + } else if(className.equalsIgnoreCase(additiveComputedField.getClass().getSimpleName())) { + return additiveComputedField; + + } else { + throw new RuntimeException("Computer field not found for className "+className); + } + + } + +} diff --git a/dashboard-analytics/src/main/java/com/tarento/analytics/helper/ComputedFieldHelper.java b/dashboard-analytics/src/main/java/com/tarento/analytics/helper/ComputedFieldHelper.java new file mode 100644 index 000000000..43c2a0762 --- /dev/null +++ b/dashboard-analytics/src/main/java/com/tarento/analytics/helper/ComputedFieldHelper.java @@ -0,0 +1,61 @@ +package com.tarento.analytics.helper; + +import com.tarento.analytics.dto.AggregateRequestDto; +import com.tarento.analytics.dto.Data; +import com.tarento.analytics.dto.Plot; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +import java.util.Map; +import java.util.function.Function; +import java.util.stream.Collectors; + +@Component +public class ComputedFieldHelper { + + public static final Logger logger = LoggerFactory.getLogger(ComputedFieldHelper.class); + + @Autowired + private ComputeHelperFactory computeHelperFactory; + + private AggregateRequestDto aggregateRequestDto; + private String postAggrTheoryName; + + public void set(AggregateRequestDto requestDto, String postAggrTheoryName){ + + this.aggregateRequestDto = requestDto; + this.postAggrTheoryName = postAggrTheoryName; + } + + public void add(Data data, String newfield, String partField, String wholeField){ + try { + Map plotMap = data.getPlots().stream().parallel().collect(Collectors.toMap(Plot::getName, Function.identity())); + + if (plotMap.get(partField).getValue() == 0.0 || plotMap.get(wholeField).getValue() == 0.0) { + + data.getPlots().add(new Plot(newfield, 0.0, "percentage")); + } else { + double wholeValue = plotMap.get(wholeField).getValue(); + double fieldValue = plotMap.get(partField).getValue() / plotMap.get(wholeField).getValue() * 100; + + if(postAggrTheoryName != null && !postAggrTheoryName.isEmpty()) { + //logger.info("Chart name: "+aggregateRequestDto.getVisualizationCode()+" :: postAggrTheoryName : "+postAggrTheoryName); + ComputeHelper computeHelper = computeHelperFactory.getInstance(postAggrTheoryName); + fieldValue = computeHelper.compute(aggregateRequestDto, fieldValue); + wholeValue = computeHelper.compute(aggregateRequestDto, wholeValue); + + } + data.getPlots().stream().filter(plot -> wholeField.equalsIgnoreCase(plot.getName())).findAny().orElse(null).setValue(wholeValue); + data.getPlots().add(new Plot(newfield, fieldValue, "percentage")); + + } + + + } catch (Exception e) { + data.getPlots().add(new Plot(newfield, 0.0, "percentage")); + } + } + +} diff --git a/dashboard-analytics/src/main/java/com/tarento/analytics/helper/IComputedField.java b/dashboard-analytics/src/main/java/com/tarento/analytics/helper/IComputedField.java new file mode 100644 index 000000000..79fcd6d6e --- /dev/null +++ b/dashboard-analytics/src/main/java/com/tarento/analytics/helper/IComputedField.java @@ -0,0 +1,12 @@ +package com.tarento.analytics.helper; + +import com.tarento.analytics.dto.AggregateRequestDto; + +import java.util.List; + +public interface IComputedField { + + public void set(AggregateRequestDto requestDto, String postAggrTheoryName); + public void add(T data, List fields, String newField); + +} diff --git a/dashboard-analytics/src/main/java/com/tarento/analytics/helper/PercentageComputedField.java b/dashboard-analytics/src/main/java/com/tarento/analytics/helper/PercentageComputedField.java new file mode 100644 index 000000000..c40ed21e0 --- /dev/null +++ b/dashboard-analytics/src/main/java/com/tarento/analytics/helper/PercentageComputedField.java @@ -0,0 +1,63 @@ +package com.tarento.analytics.helper; + +import com.tarento.analytics.dto.AggregateRequestDto; +import com.tarento.analytics.dto.Data; +import com.tarento.analytics.dto.Plot; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +import java.util.List; +import java.util.Map; +import java.util.function.Function; +import java.util.stream.Collectors; + +@Component +public class PercentageComputedField implements IComputedField{ + + public static final Logger logger = LoggerFactory.getLogger(PercentageComputedField.class); + + + private String postAggrTheoryName; + private AggregateRequestDto aggregateRequestDto; + @Autowired + private ComputeHelperFactory computeHelperFactory; + + @Override + public void set(AggregateRequestDto requestDto, String postAggrTheoryName){ + this.aggregateRequestDto = requestDto; + this.postAggrTheoryName = postAggrTheoryName; + } + + @Override + public void add(Data data, List fields, String newField) { + try { + Map plotMap = data.getPlots().stream().parallel().collect(Collectors.toMap(Plot::getName, Function.identity())); + + if (plotMap.get(fields.get(0)).getValue() == 0.0 || plotMap.get(fields.get(1)).getValue() == 0.0) { + + data.getPlots().add(new Plot(newField, 0.0, "percentage")); + } else { + double wholeValue = plotMap.get(fields.get(1)).getValue(); + double fieldValue = plotMap.get(fields.get(0)).getValue() / plotMap.get(fields.get(1)).getValue() * 100; + + + if(postAggrTheoryName != null && !postAggrTheoryName.isEmpty()) { + ComputeHelper computeHelper = computeHelperFactory.getInstance(postAggrTheoryName); + fieldValue = computeHelper.compute(aggregateRequestDto, fieldValue); + wholeValue = computeHelper.compute(aggregateRequestDto, wholeValue); + } + data.getPlots().stream().filter(plot -> fields.get(1).equalsIgnoreCase(plot.getName())).findAny().orElse(null).setValue(wholeValue); + data.getPlots().add(new Plot(newField, fieldValue, "percentage")); + } + + } catch (Exception e) { + // throw new RuntimeException("Computed field configuration not correctly provided"); + logger.error("percentage could not be computed " +e.getMessage()); + data.getPlots().add(new Plot(newField, 0.0, "percentage")); + } + + } +} + diff --git a/dashboard-analytics/src/main/java/com/tarento/analytics/helper/TargetPerDateComputeHelper.java b/dashboard-analytics/src/main/java/com/tarento/analytics/helper/TargetPerDateComputeHelper.java new file mode 100644 index 000000000..e69ed10ec --- /dev/null +++ b/dashboard-analytics/src/main/java/com/tarento/analytics/helper/TargetPerDateComputeHelper.java @@ -0,0 +1,89 @@ +package com.tarento.analytics.helper; + +import java.util.Calendar; +import java.util.Date; +import java.util.List; +import java.util.concurrent.TimeUnit; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Component; + +import com.tarento.analytics.dto.AggregateRequestDto; +import com.tarento.analytics.dto.Data; + +/** + * This implementation of Compute Helper is used to compute the difference of dates between the Request Date + * The difference is then multiplied against the Per Day Unit of Target which has been obtained from Elastic Search + * @author darshan + * + */ +@Component +public class TargetPerDateComputeHelper implements ComputeHelper { + private static final Long ROUND_OFF= 19801000l; + private static final Long NUMBER_OF_DAYS = 365l; + private static final Long LAST_HOUR = 23l; + private static final Long LAST_MINUTE = 59l; + public static final Logger logger = LoggerFactory.getLogger(TargetPerDateComputeHelper.class); + @Override + public List compute(AggregateRequestDto request, List data) { + if(request.getRequestDate()!= null && request.getRequestDate().getStartDate() != null && request.getRequestDate().getEndDate() !=null) { + try { + Long sDate = Long.parseLong(request.getRequestDate().getStartDate()); + logger.info("Start Date : " + String.valueOf(sDate)); + Long eDate = Long.parseLong(request.getRequestDate().getEndDate()); + logger.info("End Date : " + String.valueOf(eDate)); + Calendar cal = Calendar.getInstance(); + cal.setTime(new Date(eDate)); + if(cal.get(Calendar.HOUR_OF_DAY) == LAST_HOUR && cal.get(Calendar.MINUTE) == LAST_MINUTE) { + eDate = eDate + ROUND_OFF; + } + logger.info("End Date after Round Off: " + String.valueOf(eDate)); + Long dateDifference = TimeUnit.DAYS.convert((eDate - sDate), TimeUnit.MILLISECONDS); + if(dateDifference == 0l) dateDifference = dateDifference + 1l ; + for(Data eachData : data) { + Double value = (Double) eachData.getHeaderValue(); + logger.info("Value is : " + value + " :: Date Difference is : " + dateDifference); + value = (value / NUMBER_OF_DAYS) * dateDifference; + eachData.setHeaderValue(value); + } + } catch (Exception ex) { + logger.error("Encountered an error while computing the logic in Target Date Computer : " + ex.getMessage()); + } + } + return data; + } + + @Override + public Double compute(AggregateRequestDto request, double value){ + + if(request.getRequestDate()!= null && request.getRequestDate().getStartDate() != null && request.getRequestDate().getEndDate() !=null) { + try { + Long sDate = Long.parseLong(request.getRequestDate().getStartDate()); + logger.info("Start Date : " + String.valueOf(sDate)); + Long eDate = Long.parseLong(request.getRequestDate().getEndDate()); + logger.info("End Date : " + String.valueOf(eDate)); + Calendar cal = Calendar.getInstance(); + cal.setTime(new Date(eDate)); + if(cal.get(Calendar.HOUR_OF_DAY) == LAST_HOUR && cal.get(Calendar.MINUTE) == LAST_MINUTE) { + eDate = eDate + ROUND_OFF; + } + logger.info("End Date after Round Off: " + String.valueOf(eDate)); + Long dateDifference = TimeUnit.DAYS.convert((eDate - sDate), TimeUnit.MILLISECONDS); + if(dateDifference == 0l) dateDifference = dateDifference + 1l ; + + value = (value / NUMBER_OF_DAYS) * dateDifference; + logger.info("Value is : " + value + " :: Date Difference is : " + dateDifference); + + } catch (Exception ex) { + logger.error("Encountered an error while computing the logic in Target Date Computer : " + ex.getMessage()); + } + } + + return value; + + + + } + +} diff --git a/dashboard-analytics/src/main/java/com/tarento/analytics/model/AmazonS3Config.java b/dashboard-analytics/src/main/java/com/tarento/analytics/model/AmazonS3Config.java new file mode 100644 index 000000000..dbf72f9f1 --- /dev/null +++ b/dashboard-analytics/src/main/java/com/tarento/analytics/model/AmazonS3Config.java @@ -0,0 +1,52 @@ +package com.tarento.analytics.model; + +import com.amazonaws.auth.AWSCredentialsProvider; +import com.amazonaws.auth.AWSStaticCredentialsProvider; +import com.amazonaws.auth.BasicAWSCredentials; +import com.amazonaws.regions.Region; +import com.amazonaws.regions.Regions; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +@Configuration +public class AmazonS3Config +{ + @Value("${aws.access.key.id}") + private String awsKeyId; + + @Value("${aws.access.key.secret}") + private String awsKeySecret; + + @Value("${aws.region}") + private String awsRegion; + + @Value("${aws.s3.audio.bucket}") + private String awsS3AudioBucket; + + @Bean(name = "awsKeyId") + public String getAWSKeyId() { + return awsKeyId; + } + + @Bean(name = "awsKeySecret") + public String getAWSKeySecret() { + return awsKeySecret; + } + + @Bean(name = "awsRegion") + public Region getAWSPollyRegion() { + return Region.getRegion(Regions.fromName(awsRegion)); + } + + @Bean(name = "awsCredentialsProvider") + public AWSCredentialsProvider getAWSCredentials() { + BasicAWSCredentials awsCredentials = new BasicAWSCredentials(this.awsKeyId, this.awsKeySecret); + return new AWSStaticCredentialsProvider(awsCredentials); + } + + @Bean(name = "awsS3AudioBucket") + public String getAWSS3AudioBucket() { + return awsS3AudioBucket; + } +} \ No newline at end of file diff --git a/dashboard-analytics/src/main/java/com/tarento/analytics/model/ComputedFields.java b/dashboard-analytics/src/main/java/com/tarento/analytics/model/ComputedFields.java new file mode 100644 index 000000000..51f60546c --- /dev/null +++ b/dashboard-analytics/src/main/java/com/tarento/analytics/model/ComputedFields.java @@ -0,0 +1,44 @@ +package com.tarento.analytics.model; + +import java.util.ArrayList; +import java.util.List; + +public class ComputedFields { + + private String postAggregationTheory; + private String actionName; + private List fields = new ArrayList<>(); + private String newField; + + public String getPostAggregationTheory() { + return postAggregationTheory; + } + + public void setPostAggregationTheory(String postAggregationTheory) { + this.postAggregationTheory = postAggregationTheory; + } + + public String getActionName() { + return actionName; + } + + public void setActionName(String actionName) { + this.actionName = actionName; + } + + public List getFields() { + return fields; + } + + public void setFields(List fields) { + this.fields = fields; + } + + public String getNewField() { + return newField; + } + + public void setNewField(String newField) { + this.newField = newField; + } +} diff --git a/dashboard-analytics/src/main/java/com/tarento/analytics/model/ElasticSearchDictator.java b/dashboard-analytics/src/main/java/com/tarento/analytics/model/ElasticSearchDictator.java new file mode 100644 index 000000000..ec51d3054 --- /dev/null +++ b/dashboard-analytics/src/main/java/com/tarento/analytics/model/ElasticSearchDictator.java @@ -0,0 +1,69 @@ +package com.tarento.analytics.model; + +import java.util.LinkedList; +import java.util.List; +import java.util.Map; + +/** + * + * @author Darshan + * + */ +public class ElasticSearchDictator { + + private String visualisationName; + private String indexName; + private String documentType; + private Map>> queryMap; + private Map>>> aggregationMap; + private Map queryAggregationMap; + + public Map getQueryAggregationMap() { + return queryAggregationMap; + } + + public void setQueryAggregationMap(Map queryAggregationMap) { + this.queryAggregationMap = queryAggregationMap; + } + + public String getVisualisationName() { + return visualisationName; + } + + public void setVisualisationName(String visualisationName) { + this.visualisationName = visualisationName; + } + + public String getIndexName() { + return indexName; + } + + public void setIndexName(String indexName) { + this.indexName = indexName; + } + + public String getDocumentType() { + return documentType; + } + + public void setDocumentType(String documentType) { + this.documentType = documentType; + } + + public Map>> getQueryMap() { + return queryMap; + } + + public void setQueryMap(Map>> queryMap) { + this.queryMap = queryMap; + } + + public Map>>> getAggregationMap() { + return aggregationMap; + } + + public void setAggregationMap(Map>>> aggregationMap) { + this.aggregationMap = aggregationMap; + } + +} diff --git a/dashboard-analytics/src/main/java/com/tarento/analytics/model/InsightsConfiguration.java b/dashboard-analytics/src/main/java/com/tarento/analytics/model/InsightsConfiguration.java new file mode 100644 index 000000000..feae201b7 --- /dev/null +++ b/dashboard-analytics/src/main/java/com/tarento/analytics/model/InsightsConfiguration.java @@ -0,0 +1,54 @@ +package com.tarento.analytics.model; + +public class InsightsConfiguration { + private String chartResponseMap; + private String action; + private String upwardIndicator; + private String downwardIndicator; + private String textMessage; + private String colorCode; + private String insightInterval; + + public String getInsightInterval() { + return insightInterval; + } + public void setInsightInterval(String insightInterval) { + this.insightInterval = insightInterval; + } + public String getChartResponseMap() { + return chartResponseMap; + } + public void setChartResponseMap(String chartResponseMap) { + this.chartResponseMap = chartResponseMap; + } + public String getAction() { + return action; + } + public void setAction(String action) { + this.action = action; + } + public String getUpwardIndicator() { + return upwardIndicator; + } + public void setUpwardIndicator(String upwardIndicator) { + this.upwardIndicator = upwardIndicator; + } + public String getDownwardIndicator() { + return downwardIndicator; + } + public void setDownwardIndicator(String downwardIndicator) { + this.downwardIndicator = downwardIndicator; + } + public String getTextMessage() { + return textMessage; + } + public void setTextMessage(String textMessage) { + this.textMessage = textMessage; + } + public String getColorCode() { + return colorCode; + } + public void setColorCode(String colorCode) { + this.colorCode = colorCode; + } +} diff --git a/dashboard-analytics/src/main/java/com/tarento/analytics/model/Item.java b/dashboard-analytics/src/main/java/com/tarento/analytics/model/Item.java new file mode 100644 index 000000000..b1e63e89e --- /dev/null +++ b/dashboard-analytics/src/main/java/com/tarento/analytics/model/Item.java @@ -0,0 +1,92 @@ +package com.tarento.analytics.model; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; + +@JsonInclude(JsonInclude.Include.NON_NULL) +public class Item { + + @JsonProperty("name") + private String name; + @JsonProperty("code") + private String code; + @JsonProperty("id") + private Long id; + @JsonProperty("description") + private String description; + @JsonProperty("price") + private Double price; + @JsonProperty("categoryId") + private Long categoryId; + @JsonProperty("subCategoryId") + private Long subCategoryId; + @JsonProperty("name") + + + public String getName() { + return name; + } + + public Long getCategoryId() { + return categoryId; + } + + public void setCategoryId(Long categoryId) { + this.categoryId = categoryId; + } + + public Long getSubCategoryId() { + return subCategoryId; + } + + public void setSubCategoryId(Long subCategoryId) { + this.subCategoryId = subCategoryId; + } + + @JsonProperty("name") + public void setName(String name) { + this.name = name; + } + + @JsonProperty("code") + public String getCode() { + return code; + } + + @JsonProperty("code") + public void setCode(String code) { + this.code = code; + } + + @JsonProperty("id") + public Long getId() { + return id; + } + + @JsonProperty("id") + public void setId(Long id) { + this.id = id; + } + + @JsonProperty("description") + public String getDescription() { + return description; + } + + @JsonProperty("description") + public void setDescription(String description) { + this.description = description; + } + + @JsonProperty("price") + public Double getPrice() { + return price; + } + + @JsonProperty("price") + public void setPrice(Double price) { + this.price = price; + } + + +} \ No newline at end of file diff --git a/dashboard-analytics/src/main/java/com/tarento/analytics/model/KeyData.java b/dashboard-analytics/src/main/java/com/tarento/analytics/model/KeyData.java new file mode 100644 index 000000000..6a49dd3bd --- /dev/null +++ b/dashboard-analytics/src/main/java/com/tarento/analytics/model/KeyData.java @@ -0,0 +1,53 @@ + +package com.tarento.analytics.model; + +public class KeyData { + + + private Long id; + private Object key; + private String label; + + public Long getId() { + return id; + } + public void setId(Long id) { + this.id = id; + } + public Object getKey() { + return key; + } + public void setKey(Object key) { + this.key = key; + } + public String getLabel() { + return label; + } + public void setLabel(String label) { + this.label = label; + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + ((key == null) ? 0 : key.hashCode()); + return result; + } + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + KeyData other = (KeyData) obj; + if (key == null) { + if (other.key != null) + return false; + } else if (!key.equals(other.key)) + return false; + return true; + } +} diff --git a/dashboard-analytics/src/main/java/com/tarento/analytics/model/Label.java b/dashboard-analytics/src/main/java/com/tarento/analytics/model/Label.java new file mode 100644 index 000000000..d0ef0c8e9 --- /dev/null +++ b/dashboard-analytics/src/main/java/com/tarento/analytics/model/Label.java @@ -0,0 +1,19 @@ +package com.tarento.analytics.model; + +public class Label { + + private String xLabel; + private String yLabel; + public String getxLabel() { + return xLabel; + } + public void setxLabel(String xLabel) { + this.xLabel = xLabel; + } + public String getyLabel() { + return yLabel; + } + public void setyLabel(String yLabel) { + this.yLabel = yLabel; + } +} diff --git a/dashboard-analytics/src/main/java/com/tarento/analytics/model/LineData.java b/dashboard-analytics/src/main/java/com/tarento/analytics/model/LineData.java new file mode 100644 index 000000000..532eeb814 --- /dev/null +++ b/dashboard-analytics/src/main/java/com/tarento/analytics/model/LineData.java @@ -0,0 +1,20 @@ +package com.tarento.analytics.model; + +public class LineData { + + private String type; + private Object value; + public String getType() { + return type; + } + public void setType(String type) { + this.type = type; + } + public Object getValue() { + return value; + } + public void setValue(Object value) { + this.value = value; + } + +} diff --git a/dashboard-analytics/src/main/java/com/tarento/analytics/model/Query.java b/dashboard-analytics/src/main/java/com/tarento/analytics/model/Query.java new file mode 100644 index 000000000..bf34cdbc0 --- /dev/null +++ b/dashboard-analytics/src/main/java/com/tarento/analytics/model/Query.java @@ -0,0 +1,26 @@ +package com.tarento.analytics.model; + +import java.util.Map; + +public class Query { + + private String dateFilterField; + private Map aggregation; + + public String getDateFilterField() { + return dateFilterField; + } + + public void setDateFilterField(String dateFilterField) { + this.dateFilterField = dateFilterField; + } + + public Map getAggregation() { + return aggregation; + } + + public void setAggregation(Map aggregation) { + this.aggregation = aggregation; + } + +} diff --git a/dashboard-analytics/src/main/java/com/tarento/analytics/model/ServiceQuery.java b/dashboard-analytics/src/main/java/com/tarento/analytics/model/ServiceQuery.java new file mode 100644 index 000000000..f6fdd64ed --- /dev/null +++ b/dashboard-analytics/src/main/java/com/tarento/analytics/model/ServiceQuery.java @@ -0,0 +1,27 @@ +package com.tarento.analytics.model; + +public class ServiceQuery { + + private String indexName; + private String queryJson; + private String documentType; + public String getIndexName() { + return indexName; + } + public void setIndexName(String indexName) { + this.indexName = indexName; + } + public String getQueryJson() { + return queryJson; + } + public void setQueryJson(String queryJson) { + this.queryJson = queryJson; + } + public String getDocumentType() { + return documentType; + } + public void setDocumentType(String documentType) { + this.documentType = documentType; + } + +} diff --git a/dashboard-analytics/src/main/java/com/tarento/analytics/model/Transaction.java b/dashboard-analytics/src/main/java/com/tarento/analytics/model/Transaction.java new file mode 100644 index 000000000..5e5b063df --- /dev/null +++ b/dashboard-analytics/src/main/java/com/tarento/analytics/model/Transaction.java @@ -0,0 +1,264 @@ +package com.tarento.analytics.model; + +import java.util.List; + +import com.fasterxml.jackson.annotation.JsonProperty; + +/** + * Transaction Object which gets posted to Elastic Search + * @author Darshan Nagesh + * + */ +public class Transaction { + + @JsonProperty("transactionId") + private Long transactionId; + @JsonProperty("receiptNo") + private String receiptNo; + @JsonProperty("transType") + private String transType; + @JsonProperty("storeCode") + private Long storeCode; + @JsonProperty("posTerminalNo") + private String posTerminalNo; + @JsonProperty("staffId") + private Long staffId; + @JsonProperty("transDate") + private String transDate; + @JsonProperty("transTime") + private String transTime; + @JsonProperty("customerNo") + private Long consumerNo; + @JsonProperty("salesType") + private String salesType; + @JsonProperty("netAmt") + private Double netAmt; + @JsonProperty("grossAmt") + private Double grossAmt; + @JsonProperty("payment") + private Double payment; + @JsonProperty("discountAmt") + private Double discountAmt; + @JsonProperty("costAmt") + private Double costAmt; + @JsonProperty("managerId") + private Long managerId; + @JsonProperty("isTraining") + private Boolean isTraining; + @JsonProperty("statementNo") + private String statementNo; + @JsonProperty("postingStatus") + private String postingStatus; + @JsonProperty("postStatementNo") + private String postStatementNo; + @JsonProperty("customerAgeGroup") + private String customerAgeGroup; + @JsonProperty("customerGender") + private String customerGender; + @JsonProperty("items") + private List items; + @JsonProperty("itemDetails") + private List itemDetails; + @JsonProperty("orgId") + private Long orgId; + + public Long getOrgId() { + return orgId; + } + + public void setOrgId(Long orgId) { + this.orgId = orgId; + } + + public List getItemDetails() { + return itemDetails; + } + + public void setItemDetails(List itemDetails) { + this.itemDetails = itemDetails; + } + + public List getItems() { + return items; + } + + public void setItems(List items) { + this.items = items; + } + + public String getCustomerAgeGroup() { + return customerAgeGroup; + } + + public void setCustomerAgeGroup(String customerAgeGroup) { + this.customerAgeGroup = customerAgeGroup; + } + + public String getCustomerGender() { + return customerGender; + } + + public void setCustomerGender(String customerGender) { + this.customerGender = customerGender; + } + + public Long getTransactionId() { + return transactionId; + } + + public void setTransactionId(Long transactionId) { + this.transactionId = transactionId; + } + + public String getReceiptNo() { + return receiptNo; + } + + public void setReceiptNo(String receiptNo) { + this.receiptNo = receiptNo; + } + + public String getTransType() { + return transType; + } + + public void setTransType(String transType) { + this.transType = transType; + } + + public Long getStoreCode() { + return storeCode; + } + + public void setStoreCode(Long storeCode) { + this.storeCode = storeCode; + } + + public String getPosTerminalNo() { + return posTerminalNo; + } + + public void setPosTerminalNo(String posTerminalNo) { + this.posTerminalNo = posTerminalNo; + } + + public Long getStaffId() { + return staffId; + } + + public void setStaffId(Long staffId) { + this.staffId = staffId; + } + + public String getTransDate() { + return transDate; + } + + public void setTransDate(String transDate) { + this.transDate = transDate; + } + + public String getTransTime() { + return transTime; + } + + public void setTransTime(String transTime) { + this.transTime = transTime; + } + + public Long getConsumerNo() { + return consumerNo; + } + + public void setConsumerNo(Long consumerNo) { + this.consumerNo = consumerNo; + } + + public String getSalesType() { + return salesType; + } + + public void setSalesType(String salesType) { + this.salesType = salesType; + } + + public Double getNetAmt() { + return netAmt; + } + + public void setNetAmt(Double netAmt) { + this.netAmt = netAmt; + } + + public Double getGrossAmt() { + return grossAmt; + } + + public void setGrossAmt(Double grossAmt) { + this.grossAmt = grossAmt; + } + + public Double getPayment() { + return payment; + } + + public void setPayment(Double payment) { + this.payment = payment; + } + + public Double getDiscountAmt() { + return discountAmt; + } + + public void setDiscountAmt(Double discountAmt) { + this.discountAmt = discountAmt; + } + + public Double getCostAmt() { + return costAmt; + } + + public void setCostAmt(Double costAmt) { + this.costAmt = costAmt; + } + + public Long getManagerId() { + return managerId; + } + + public void setManagerId(Long managerId) { + this.managerId = managerId; + } + + public Boolean getIsTraining() { + return isTraining; + } + + public void setIsTraining(Boolean isTraining) { + this.isTraining = isTraining; + } + + public String getStatementNo() { + return statementNo; + } + + public void setStatementNo(String statementNo) { + this.statementNo = statementNo; + } + + public String getPostingStatus() { + return postingStatus; + } + + public void setPostingStatus(String postingStatus) { + this.postingStatus = postingStatus; + } + + public String getPostStatementNo() { + return postStatementNo; + } + + public void setPostStatementNo(String postStatementNo) { + this.postStatementNo = postStatementNo; + } +} diff --git a/dashboard-analytics/src/main/java/com/tarento/analytics/model/dashboardConfig/Chart.java b/dashboard-analytics/src/main/java/com/tarento/analytics/model/dashboardConfig/Chart.java new file mode 100644 index 000000000..9a83dbf1e --- /dev/null +++ b/dashboard-analytics/src/main/java/com/tarento/analytics/model/dashboardConfig/Chart.java @@ -0,0 +1,108 @@ +package com.tarento.analytics.model.dashboardConfig; + +import java.util.List; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; + +@JsonInclude(JsonInclude.Include.NON_NULL) +public class Chart { + +@JsonProperty("id") +private Long id; +@JsonProperty("name") +private String name; +@JsonProperty("code") +private String code; +@JsonProperty("serviceApi") +private String serviceApi; +@JsonProperty("chartType") +private String chartType; +@JsonProperty("headersAvailable") +private Boolean headersAvailable; +@JsonProperty("filter") +private String filter; +@JsonProperty("headers") +private List
headers = null; + + +public String getFilter() { + return filter; +} + +public void setFilter(String filter) { + this.filter = filter; +} + +@JsonProperty("id") +public Long getId() { +return id; +} + +@JsonProperty("id") +public void setId(Long id) { +this.id = id; +} + +@JsonProperty("name") +public String getName() { +return name; +} + +@JsonProperty("name") +public void setName(String name) { +this.name = name; +} + +@JsonProperty("code") +public String getCode() { +return code; +} + +@JsonProperty("code") +public void setCode(String code) { +this.code = code; +} + +@JsonProperty("serviceApi") +public String getServiceApi() { +return serviceApi; +} + +@JsonProperty("serviceApi") +public void setServiceApi(String serviceApi) { +this.serviceApi = serviceApi; +} + +@JsonProperty("chartType") +public String getChartType() { +return chartType; +} + +@JsonProperty("chartType") +public void setChartType(String chartType) { +this.chartType = chartType; +} + +@JsonProperty("headersAvailable") +public Boolean getHeadersAvailable() { +return headersAvailable; +} + +@JsonProperty("headersAvailable") +public void setHeadersAvailable(Boolean headersAvailable) { +this.headersAvailable = headersAvailable; +} + +@JsonProperty("headers") +public List
getHeaders() { +return headers; +} + +@JsonProperty("headers") +public void setHeaders(List
headers) { +this.headers = headers; +} + + +} \ No newline at end of file diff --git a/dashboard-analytics/src/main/java/com/tarento/analytics/model/dashboardConfig/Dashboard.java b/dashboard-analytics/src/main/java/com/tarento/analytics/model/dashboardConfig/Dashboard.java new file mode 100644 index 000000000..f06271f76 --- /dev/null +++ b/dashboard-analytics/src/main/java/com/tarento/analytics/model/dashboardConfig/Dashboard.java @@ -0,0 +1,100 @@ +package com.tarento.analytics.model.dashboardConfig; + +import java.util.List; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.tarento.analytics.dto.DashboardDto; + +@JsonInclude(JsonInclude.Include.NON_NULL) +public class Dashboard { + +@JsonProperty("id") +private Long id; +@JsonProperty("name") +private String name; +@JsonProperty("code") +private String code; +@JsonProperty("isActive") +private Boolean isActive; +@JsonProperty("description") +private String description; +@JsonProperty("placement") +private String placement; +@JsonProperty("visualizations") +private List visualizations = null; + +public Dashboard() {} +public Dashboard(DashboardDto dto) { + this.id = dto.getId(); + this.name = dto.getName(); + this.code = dto.getCode(); + this.description = dto.getDescription(); + this.placement = dto.getPlacement(); +} + +public String getPlacement() { + return placement; +} +public void setPlacement(String placement) { + this.placement = placement; +} +public Long getId() { + return id; +} + +public void setId(Long id) { + this.id = id; +} + +@JsonProperty("name") +public String getName() { +return name; +} + +@JsonProperty("name") +public void setName(String name) { +this.name = name; +} + +@JsonProperty("code") +public String getCode() { +return code; +} + +@JsonProperty("code") +public void setCode(String code) { +this.code = code; +} + +@JsonProperty("isActive") +public Boolean getIsActive() { +return isActive; +} + +@JsonProperty("isActive") +public void setIsActive(Boolean isActive) { +this.isActive = isActive; +} + +@JsonProperty("description") +public String getDescription() { +return description; +} + +@JsonProperty("description") +public void setDescription(String description) { +this.description = description; +} + +@JsonProperty("visualizations") +public List getVisualizations() { +return visualizations; +} + +@JsonProperty("visualizations") +public void setVisualizations(List visualizations) { +this.visualizations = visualizations; +} + +} \ No newline at end of file diff --git a/dashboard-analytics/src/main/java/com/tarento/analytics/model/dashboardConfig/Header.java b/dashboard-analytics/src/main/java/com/tarento/analytics/model/dashboardConfig/Header.java new file mode 100644 index 000000000..60572bbf9 --- /dev/null +++ b/dashboard-analytics/src/main/java/com/tarento/analytics/model/dashboardConfig/Header.java @@ -0,0 +1,55 @@ +package com.tarento.analytics.model.dashboardConfig; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; + +@JsonInclude(JsonInclude.Include.NON_NULL) +public class Header { + +@JsonProperty("id") +private Long id; +@JsonProperty("label") +private String label; +@JsonProperty("data") +private String data; +@JsonProperty("field") +private String field; + + +public Long getId() { + return id; +} + +public void setId(Long id) { + this.id = id; +} + +public String getField() { + return field; +} + +public void setField(String field) { + this.field = field; +} + +@JsonProperty("label") +public String getLabel() { +return label; +} + +@JsonProperty("label") +public void setLabel(String label) { +this.label = label; +} + +@JsonProperty("data") +public String getData() { +return data; +} + +@JsonProperty("data") +public void setData(String data) { +this.data = data; +} + +} \ No newline at end of file diff --git a/dashboard-analytics/src/main/java/com/tarento/analytics/model/dashboardConfig/Visualization.java b/dashboard-analytics/src/main/java/com/tarento/analytics/model/dashboardConfig/Visualization.java new file mode 100644 index 000000000..f88286ff8 --- /dev/null +++ b/dashboard-analytics/src/main/java/com/tarento/analytics/model/dashboardConfig/Visualization.java @@ -0,0 +1,114 @@ +package com.tarento.analytics.model.dashboardConfig; + +import java.util.List; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; + +@JsonInclude(JsonInclude.Include.NON_NULL) +public class Visualization { + +@JsonProperty("id") +private Long id; +@JsonProperty("name") +private String name; +@JsonProperty("description") +private String description; +@JsonProperty("chartType") +private String chartType; +@JsonProperty("charts") +private List charts = null; +private List storeId; +@JsonProperty("salesAreaCode") +private String salesAreaCode; +@JsonProperty("countryCode") +private String countryCode; + + +public List getStoreId() { + return storeId; +} + + +@JsonProperty("visualRank") +private Long visualRank; + + +public void setStoreId(List storeId) { + this.storeId = storeId; +} + +public String getSalesAreaCode() { + return salesAreaCode; +} + +public void setSalesAreaCode(String salesAreaCode) { + this.salesAreaCode = salesAreaCode; +} + +public String getCountryCode() { + return countryCode; +} + +public void setCountryCode(String countryCode) { + this.countryCode = countryCode; +} + +public Long getVisualRank() { + return visualRank; +} + +public void setVisualRank(Long visualRank) { + this.visualRank = visualRank; +} + +public String getChartType() { + return chartType; +} + +public void setChartType(String chartType) { + this.chartType = chartType; +} + +@JsonProperty("id") +public Long getId() { +return id; +} + +@JsonProperty("id") +public void setId(Long id) { +this.id = id; +} + +@JsonProperty("name") +public String getName() { +return name; +} + +@JsonProperty("name") +public void setName(String name) { +this.name = name; +} + +@JsonProperty("description") +public String getDescription() { +return description; +} + +@JsonProperty("description") +public void setDescription(String description) { +this.description = description; +} + +@JsonProperty("charts") +public List getCharts() { +return charts; +} + +@JsonProperty("charts") +public void setCharts(List charts) { +this.charts = charts; +} + + +} \ No newline at end of file diff --git a/dashboard-analytics/src/main/java/com/tarento/analytics/org/service/ClientService.java b/dashboard-analytics/src/main/java/com/tarento/analytics/org/service/ClientService.java new file mode 100644 index 000000000..813448ecb --- /dev/null +++ b/dashboard-analytics/src/main/java/com/tarento/analytics/org/service/ClientService.java @@ -0,0 +1,47 @@ +package com.tarento.analytics.org.service; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.tarento.analytics.constant.Constants; +import com.tarento.analytics.dto.*; +import com.tarento.analytics.exception.AINException; + +public interface ClientService { + + public AggregateDto getAggregatedData(AggregateRequestDto req, List roles) throws AINException, IOException; + public List getHeaderData(CummulativeDataRequestDto requestDto, List roles) throws AINException; + + /** + * Default handle request to translates tenant code to a value + */ + default boolean preHandle(AggregateRequestDto request, ObjectNode chartNode, MdmsApiMappings mdmsApiMappings){ + boolean status = Boolean.FALSE; + JsonNode translateNode = chartNode.get(Constants.JsonPaths.TRANSLATE_CODE); + boolean isTranslate = translateNode == null || translateNode.asBoolean()==Boolean.FALSE ? Boolean.FALSE : translateNode.asBoolean(); + boolean valueExist = request.getFilters().containsKey(Constants.MDMSKeys.TENANT_ID); + if(valueExist && isTranslate){ + Object filter = request.getFilters().get(Constants.MDMSKeys.TENANT_ID); + List values = new ArrayList<>(); + if(filter instanceof ArrayList){ + for(Object code : ((ArrayList)request.getFilters().get(Constants.MDMSKeys.TENANT_ID))){ + values.add(mdmsApiMappings.valueOf(code.toString())); + } + request.getFilters().put(Constants.MDMSKeys.TENANT_ID, values); + status = Boolean.TRUE; + + } else if(filter instanceof String){ + String code = request.getFilters().get(Constants.MDMSKeys.TENANT_ID).toString(); + String value = mdmsApiMappings.valueOf(code); + request.getFilters().put(Constants.MDMSKeys.TENANT_ID, value); + status = Boolean.TRUE; + } + + } + return isTranslate; + } + +} diff --git a/dashboard-analytics/src/main/java/com/tarento/analytics/org/service/ClientServiceFactory.java b/dashboard-analytics/src/main/java/com/tarento/analytics/org/service/ClientServiceFactory.java new file mode 100644 index 000000000..0ec5fcdb6 --- /dev/null +++ b/dashboard-analytics/src/main/java/com/tarento/analytics/org/service/ClientServiceFactory.java @@ -0,0 +1,46 @@ +package com.tarento.analytics.org.service; + +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.tarento.analytics.ConfigurationLoader; +import com.tarento.analytics.constant.Constants; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +@Component +public class ClientServiceFactory { + + @Autowired + TarentoServiceImpl tarentoServiceImpl; + + @Autowired + MdmsServiceImpl mdmsServiceImpl; + + @Autowired + ConfigurationLoader configurationLoader; + + public ClientService getInstance(Constants.ClienServiceType clientServiceName){ + + if(clientServiceName.equals(Constants.ClienServiceType.DEFAULT_CLIENT)) + return tarentoServiceImpl; + else if(clientServiceName.equals(Constants.ClienServiceType.MDMS_CLIENT)) + return mdmsServiceImpl; + + else + throw new RuntimeException(clientServiceName + "not found"); + + } + + public ClientService get(String chartId){ + + ObjectNode node = configurationLoader.get(Constants.ConfigurationFiles.CHART_API_CONFIG); + ObjectNode chartNode = (ObjectNode) node.get(chartId); + boolean mdmsEnable = chartNode.get(Constants.JsonPaths.IS_MDMS_ENABALED) == null ? false : chartNode.get(Constants.JsonPaths.IS_MDMS_ENABALED).asBoolean(); + + if(mdmsEnable) + return mdmsServiceImpl; + else + return tarentoServiceImpl; + + } + +} diff --git a/dashboard-analytics/src/main/java/com/tarento/analytics/org/service/MdmsApiMappings.java b/dashboard-analytics/src/main/java/com/tarento/analytics/org/service/MdmsApiMappings.java new file mode 100644 index 000000000..19a01080e --- /dev/null +++ b/dashboard-analytics/src/main/java/com/tarento/analytics/org/service/MdmsApiMappings.java @@ -0,0 +1,180 @@ +package com.tarento.analytics.org.service; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.tarento.analytics.constant.Constants; +import com.tarento.analytics.service.impl.RestService; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; + +import javax.annotation.PostConstruct; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +@Component +public class MdmsApiMappings { + + private boolean isTranslate = Boolean.FALSE; + private final String TESTING_ID = "pb.testing"; + private final String NAME = "name"; + + private static Logger logger = LoggerFactory.getLogger(MdmsApiMappings.class); + + private Map ddrTenantMapping = new HashMap<>(); + private Map> ddrTenantMapping1 = new HashMap<>(); + private Map codeValues = new HashMap<>(); + private Map> ddrValueMap = new HashMap<>(); + + @Value("${egov.mdms-service.target.url}") + private String mdmsServiceSearchUri; + + @Autowired + private RestService restService; + + @Autowired + private ObjectMapper mapper; + + @Value("${egov.mdms-service.request}") + private String REQUEST_INFO_STR ;//="{\"RequestInfo\":{\"authToken\":\"\"},\"MdmsCriteria\":{\"tenantId\":\"pb\",\"moduleDetails\":[{\"moduleName\":\"tenant\",\"masterDetails\":[{\"name\":\"tenants\"}]}]}}"; + + public String valueOf(String code){ + return codeValues.getOrDefault(code, null); + } + + public void setTranslate(boolean isTranslate){ + this.isTranslate = isTranslate; + } + + @PostConstruct + public void loadMdmsService() throws Exception { + + JsonNode requestInfo = mapper.readTree(REQUEST_INFO_STR); + try { + JsonNode response = restService.post(mdmsServiceSearchUri, "", requestInfo); + ArrayNode tenants = (ArrayNode) response.findValues(Constants.MDMSKeys.TENANTS).get(0); + + + for(JsonNode tenant : tenants) { + JsonNode tenantId = tenant.findValue(Constants.MDMSKeys.CODE); + JsonNode ddrCode = tenant.findValue(Constants.MDMSKeys.DISTRICT_CODE); + JsonNode ddrName = tenant.findValue(Constants.MDMSKeys.DDR_NAME); + + JsonNode name = tenant.findValue(NAME); + if(!codeValues.containsKey(tenantId.asText())) codeValues.put(tenantId.asText(), name.asText()); + + + if(!tenantId.asText().equalsIgnoreCase(TESTING_ID)) { + if(!ddrTenantMapping1.containsKey(ddrName.asText())){ + List tenantList = new ArrayList<>(); + tenantList.add(tenantId.asText()); + ddrTenantMapping1.put(ddrName.asText(),tenantList); + List values = new ArrayList<>(); + values.add(name.asText()); + ddrValueMap.put(ddrName.asText(), values); + + } else { + ddrTenantMapping1.get(ddrName.asText()).add(tenantId.asText()); + ddrValueMap.get(ddrName.asText()).add(name.asText()); + + } + + if (!ddrTenantMapping.containsKey(ddrCode.asText())){ + ddrTenantMapping.put(ddrCode.asText(), ddrName.asText()); + } + } + + } + } catch (Exception e){ + getDefaultMapping(); + logger.error("Loading Mdms service error: "+e.getMessage()+" :: loaded default DDRs"); + } + //logger.info("ddrTenantMapping = "+ddrTenantMapping); + logger.info("ddrTenantMapping1 = "+ddrTenantMapping1); + + } + + public String getDDRNameByCode(String ddrCode){ + return ddrTenantMapping.getOrDefault(ddrCode, ""); + } + + public List getTenantIds(String ddrCode){ + return ddrTenantMapping1.getOrDefault(ddrCode, new ArrayList<>()); + } + + public String getDDRName(String tenantId){ + + for(Map.Entry entry : isTranslate ? ddrValueMap.entrySet() :ddrTenantMapping1.entrySet()){ + List values = (List) entry.getValue(); + if(values.contains(tenantId)) return entry.getKey().toString(); + + } + return null; + + } + + public Map> getGroupedTenants(List tenants){ + + Map> groupTenantIds = new HashMap<>(); + + if(tenants!=null){ + for(String tenant : tenants) { + + String ddrName = getDDRName(tenant); + if (groupTenantIds.containsKey(ddrName)){ + groupTenantIds.get(ddrName).add(tenant); + + } else { + List tenantList = new ArrayList<>(); + tenantList.add(tenant); + groupTenantIds.put(ddrName,tenantList); + } + + } + } + + return groupTenantIds; + } + + + public Map> getAll(){ + return isTranslate ? ddrValueMap : ddrTenantMapping1; + } + + private void getDefaultMapping(){ + + ddrTenantMapping.put("1", "Amritsar-DDR"); + ddrTenantMapping.put("2", "Patiala-DDR"); + ddrTenantMapping.put("3", "Bathinda-DDR"); + ddrTenantMapping.put("4", "Ferozepur-DDR"); + ddrTenantMapping.put("5", "Ludhiana-DDR"); + ddrTenantMapping.put("6", "Ferozepur-DDR"); + ddrTenantMapping.put("7", "Ferozepur-DDR"); + ddrTenantMapping.put("8", "Amritsar-DDR"); + ddrTenantMapping.put("9", "Jalandhar-DDR"); + ddrTenantMapping.put("10", "Jalandhar-DDR"); + + ddrTenantMapping.put("11", "Jalandhar-DDR"); + ddrTenantMapping.put("12", "Ludhiana-DDR"); + ddrTenantMapping.put("13", "Bathinda-DDR"); + ddrTenantMapping.put("14", "Ferozepur-DDR"); + ddrTenantMapping.put("15", "Patiala-DDR"); + ddrTenantMapping.put("16", "Bathinda-DDR"); + ddrTenantMapping.put("17", "Jalandhar-DDR"); + ddrTenantMapping.put("18", "Pathankot-MC"); + ddrTenantMapping.put("19", "Patiala-DDR"); + ddrTenantMapping.put("20", "Ludhiana-DDR"); + ddrTenantMapping.put("21", "Patiala-DDR"); + ddrTenantMapping.put("22", "Bathinda-DDR"); + ddrTenantMapping.put("140001", "Ludhiana-DDR"); + + } + + +} + diff --git a/dashboard-analytics/src/main/java/com/tarento/analytics/org/service/MdmsService.java b/dashboard-analytics/src/main/java/com/tarento/analytics/org/service/MdmsService.java new file mode 100644 index 000000000..d23aacca1 --- /dev/null +++ b/dashboard-analytics/src/main/java/com/tarento/analytics/org/service/MdmsService.java @@ -0,0 +1,95 @@ +package com.tarento.analytics.org.service; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.tarento.analytics.constant.Constants; +import com.tarento.analytics.service.impl.RestService; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; + +import javax.annotation.PostConstruct; +import java.util.HashMap; +import java.util.Map; + +@Component +public class MdmsService { + + private static Logger logger = LoggerFactory.getLogger(MdmsService.class); + + private Map ddrTenantMapping = new HashMap<>(); + + @Value("${egov.mdms-service.target.url}") + private String mdmsServiceSearchUri; + + @Autowired + private RestService restService; + + @Autowired + private ObjectMapper mapper; + + @Value("${egov.mdms-service.request}") + private String REQUEST_INFO_STR ;//="{\"RequestInfo\":{\"authToken\":\"\"},\"MdmsCriteria\":{\"tenantId\":\"pb\",\"moduleDetails\":[{\"moduleName\":\"tenant\",\"masterDetails\":[{\"name\":\"tenants\"}]}]}}"; + + + @PostConstruct + public void loadMdmsService() throws Exception{ + + JsonNode requestInfo = mapper.readTree(REQUEST_INFO_STR); + try { + JsonNode response = restService.post(mdmsServiceSearchUri, "", requestInfo); + ArrayNode tenants = (ArrayNode) response.findValues(Constants.MDMSKeys.TENANTS).get(0); + + + for(JsonNode tenant : tenants) { + JsonNode ddrCode = tenant.findValue(Constants.MDMSKeys.DISTRICT_CODE); + JsonNode ddrName = tenant.findValue(Constants.MDMSKeys.DDR_NAME); + + if (!ddrTenantMapping.containsKey(ddrCode.asText())){ + ddrTenantMapping.put(ddrCode.asText(), ddrName.asText()); + } + } + } catch (Exception e){ + getDefaultMapping(); + logger.error("Loading Mdms service error: "+e.getMessage()+" :: loaded default DDRs"); + } + logger.info("ddrTenantMapping = "+ddrTenantMapping); + } + + public String getDDRNameByCode(String ddrCode){ + return ddrTenantMapping.getOrDefault(ddrCode, ""); + } + + private void getDefaultMapping(){ + + ddrTenantMapping.put("1", "Amritsar-DDR"); + ddrTenantMapping.put("2", "Patiala-DDR"); + ddrTenantMapping.put("3", "Bathinda-DDR"); + ddrTenantMapping.put("4", "Ferozepur-DDR"); + ddrTenantMapping.put("5", "Ludhiana-DDR"); + ddrTenantMapping.put("6", "Ferozepur-DDR"); + ddrTenantMapping.put("7", "Ferozepur-DDR"); + ddrTenantMapping.put("8", "Amritsar-DDR"); + ddrTenantMapping.put("9", "Jalandhar-DDR"); + ddrTenantMapping.put("10", "Jalandhar-DDR"); + + ddrTenantMapping.put("11", "Jalandhar-DDR"); + ddrTenantMapping.put("12", "Ludhiana-DDR"); + ddrTenantMapping.put("13", "Bathinda-DDR"); + ddrTenantMapping.put("14", "Ferozepur-DDR"); + ddrTenantMapping.put("15", "Patiala-DDR"); + ddrTenantMapping.put("16", "Bathinda-DDR"); + ddrTenantMapping.put("17", "Jalandhar-DDR"); + ddrTenantMapping.put("18", "Pathankot-MC"); + ddrTenantMapping.put("19", "Patiala-DDR"); + ddrTenantMapping.put("20", "Ludhiana-DDR"); + ddrTenantMapping.put("21", "Patiala-DDR"); + ddrTenantMapping.put("22", "Bathinda-DDR"); + ddrTenantMapping.put("140001", "Ludhiana-DDR"); + + } + +} diff --git a/dashboard-analytics/src/main/java/com/tarento/analytics/org/service/MdmsServiceImpl.java b/dashboard-analytics/src/main/java/com/tarento/analytics/org/service/MdmsServiceImpl.java new file mode 100644 index 000000000..4333fc4f1 --- /dev/null +++ b/dashboard-analytics/src/main/java/com/tarento/analytics/org/service/MdmsServiceImpl.java @@ -0,0 +1,161 @@ +package com.tarento.analytics.org.service; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.JsonNodeFactory; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.tarento.analytics.ConfigurationLoader; +import com.tarento.analytics.constant.Constants; +import com.tarento.analytics.dto.*; +import com.tarento.analytics.enums.ChartType; +import com.tarento.analytics.exception.AINException; +import com.tarento.analytics.handler.IResponseHandler; +import com.tarento.analytics.handler.ResponseHandlerFactory; +import com.tarento.analytics.service.QueryService; +import com.tarento.analytics.service.impl.RestService; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +@Component +public class MdmsServiceImpl implements ClientService { + + public static final Logger logger = LoggerFactory.getLogger(MdmsServiceImpl.class); + + @Autowired + private QueryService queryService; + + @Autowired + private RestService restService; + + @Autowired + private ConfigurationLoader configurationLoader; + + @Autowired + private ResponseHandlerFactory responseHandlerFactory; + + @Autowired + private MdmsApiMappings mdmsApiMappings; + + Map> groupTenantIds; + + @Override + public AggregateDto getAggregatedData(AggregateRequestDto request, List roles) throws AINException, IOException { + // Read visualization Code + String chartId = request.getVisualizationCode(); + logger.info("chartId >> "+chartId); + + // Load Chart API configuration to Object Node for easy retrieval later + ObjectNode node = configurationLoader.get(Constants.ConfigurationFiles.CHART_API_CONFIG); + logger.info("conf node >> "+node); + + ObjectNode chartNode = (ObjectNode) node.get(chartId); + logger.info("chartNode >> "+chartNode); + boolean isHandle = preHandle(request, chartNode, mdmsApiMappings); + mdmsApiMappings.setTranslate(isHandle); + + ChartType chartType = ChartType.fromValue(chartNode.get(Constants.JsonPaths.CHART_TYPE).asText()); + groupTenantIds = mdmsApiMappings.getGroupedTenants((List) request.getFilters().get("tenantId")); + + + ObjectNode reponseNode = buildResponse(chartNode, request); + + //replacing default values by + request.setChartNode(chartNode); + IResponseHandler responseHandler = responseHandlerFactory.getInstance(chartType); + AggregateDto aggregateDto = new AggregateDto(); + if (reponseNode.fields().hasNext()) { + + aggregateDto = responseHandler.translate(request, reponseNode); + } + return aggregateDto; + } + + public ObjectNode buildResponse(ObjectNode chartNode, AggregateRequestDto request) { + + String plotName = chartNode.get("plotLabel") == null ? "DDR" : chartNode.get("plotLabel").asText(); + ChartType chartType = ChartType.fromValue(chartNode.get(Constants.JsonPaths.CHART_TYPE).asText()); + boolean isDefaultPresent = chartType.equals(ChartType.LINE) && chartNode.get(Constants.JsonPaths.INTERVAL) != null; + boolean isRequestContainsInterval = null == request.getRequestDate() ? false : (request.getRequestDate().getInterval() != null && !request.getRequestDate().getInterval().isEmpty()); + String interval = isRequestContainsInterval ? request.getRequestDate().getInterval() : (isDefaultPresent ? chartNode.get(Constants.JsonPaths.INTERVAL).asText() : ""); + + ObjectNode aggrObjectNode = JsonNodeFactory.instance.objectNode(); + ObjectNode nodes = JsonNodeFactory.instance.objectNode(); + + ArrayNode queries = (ArrayNode) chartNode.get(Constants.JsonPaths.QUERIES); + queries.forEach(query -> { + AggregateRequestDto requestDto = request; + + + String module = query.get(Constants.JsonPaths.MODULE).asText(); + if (request.getModuleLevel().equals(Constants.Modules.HOME_REVENUE) || + request.getModuleLevel().equals(Constants.Modules.HOME_SERVICES) || + query.get(Constants.JsonPaths.MODULE).asText().equals(Constants.Modules.COMMON) || + request.getModuleLevel().equals(module)) { + + String indexName = query.get(Constants.JsonPaths.INDEX_NAME).asText(); + // intercept request + _search operation + ObjectNode aggrResponse = aggrResponseBuilder(plotName, requestDto, query, indexName, interval); + if(nodes.has(indexName)) { + indexName = indexName + "_1"; + } + nodes.set(indexName, aggrResponse); + aggrObjectNode.set(Constants.JsonPaths.AGGREGATIONS, nodes); + } + }); + return aggrObjectNode; + } + + + private ObjectNode aggrResponseBuilder(String nodeName, AggregateRequestDto requestDto, JsonNode query, String indexName, String interval) { + + ObjectNode nodes = JsonNodeFactory.instance.objectNode(); + ArrayNode bucket = JsonNodeFactory.instance.arrayNode(); + + if (groupTenantIds.size() == 0) { //no tenantId filter present + bucket = getBuckets(mdmsApiMappings.getAll(), requestDto, query, indexName); + requestDto.getFilters().clear(); + + } else { //comes from global filter + bucket = getBuckets(groupTenantIds, requestDto, query, indexName); + + } + ObjectNode buckets = JsonNodeFactory.instance.objectNode(); + buckets.set("buckets", bucket); + nodes.put(nodeName, buckets); + return nodes; + + } + + private ArrayNode getBuckets(Map> map, AggregateRequestDto requestDto, JsonNode query, String indexName) { + ObjectNode nodes = JsonNodeFactory.instance.objectNode(); + ArrayNode bucket = JsonNodeFactory.instance.arrayNode(); + for (String ddrkey : map.keySet()) { + List tenantIds = map.get(ddrkey); + requestDto.getFilters().put("tenantId", tenantIds); + ObjectNode requestNode = queryService.getChartConfigurationQuery(requestDto, query, indexName, null); + + try { + ObjectNode aggrNode = (ObjectNode) restService.search(indexName, requestNode.toString()); + + if(!ddrkey.equalsIgnoreCase("null")) bucket.add(((ObjectNode) aggrNode.get(Constants.JsonPaths.AGGREGATIONS)).put("key", ddrkey)); + } catch (Exception e) { + logger.error("Encountered an Exception while Executing the Query : " + e.getMessage()); + } + + } + return bucket; + } + + + + @Override + public List getHeaderData(CummulativeDataRequestDto requestDto, List roles) throws AINException { + return null; + } +} diff --git a/dashboard-analytics/src/main/java/com/tarento/analytics/org/service/TarentoServiceImpl.java b/dashboard-analytics/src/main/java/com/tarento/analytics/org/service/TarentoServiceImpl.java new file mode 100644 index 000000000..3193f11dc --- /dev/null +++ b/dashboard-analytics/src/main/java/com/tarento/analytics/org/service/TarentoServiceImpl.java @@ -0,0 +1,187 @@ +package com.tarento.analytics.org.service; + +import java.io.IOException; +import java.util.Calendar; +import java.util.Date; +import java.util.List; +import java.util.concurrent.TimeUnit; + +import org.apache.commons.lang3.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.JsonNodeFactory; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.tarento.analytics.ConfigurationLoader; +import com.tarento.analytics.constant.Constants; +import com.tarento.analytics.dto.AggregateDto; +import com.tarento.analytics.dto.AggregateRequestDto; +import com.tarento.analytics.dto.CummulativeDataRequestDto; +import com.tarento.analytics.dto.DashboardHeaderDto; +import com.tarento.analytics.dto.RoleDto; +import com.tarento.analytics.enums.ChartType; +import com.tarento.analytics.exception.AINException; +import com.tarento.analytics.handler.IResponseHandler; +import com.tarento.analytics.handler.InsightsHandler; +import com.tarento.analytics.handler.InsightsHandlerFactory; +import com.tarento.analytics.handler.ResponseHandlerFactory; +import com.tarento.analytics.model.InsightsConfiguration; +import com.tarento.analytics.service.QueryService; +import com.tarento.analytics.service.impl.RestService; + + +@Component +public class TarentoServiceImpl implements ClientService { + + public static final Logger logger = LoggerFactory.getLogger(TarentoServiceImpl.class); + + ObjectMapper mapper = new ObjectMapper(); + char insightPrefix = 'i'; + + + @Autowired + private QueryService queryService; + + @Autowired + private RestService restService; + + @Autowired + private ConfigurationLoader configurationLoader; + + @Autowired + private ResponseHandlerFactory responseHandlerFactory; + + @Autowired + private InsightsHandlerFactory insightsHandlerFactory; + + @Autowired + private MdmsApiMappings mdmsApiMappings; + + @Override + public AggregateDto getAggregatedData(AggregateRequestDto request, List roles) throws AINException, IOException { + // Read visualization Code + String internalChartId = request.getVisualizationCode(); + ObjectNode aggrObjectNode = JsonNodeFactory.instance.objectNode(); + ObjectNode insightAggrObjectNode = JsonNodeFactory.instance.objectNode(); + ObjectNode nodes = JsonNodeFactory.instance.objectNode(); + ObjectNode insightNodes = JsonNodeFactory.instance.objectNode(); + Boolean continueWithInsight = Boolean.FALSE; + + // Load Chart API configuration to Object Node for easy retrieval later + ObjectNode node = configurationLoader.get(Constants.ConfigurationFiles.CHART_API_CONFIG); + ObjectNode chartNode = (ObjectNode) node.get(internalChartId); + InsightsConfiguration insightsConfig = null; + if(chartNode.get(Constants.JsonPaths.INSIGHT) != null) { + insightsConfig = mapper.treeToValue(chartNode.get(Constants.JsonPaths.INSIGHT), InsightsConfiguration.class); + } + ChartType chartType = ChartType.fromValue(chartNode.get(Constants.JsonPaths.CHART_TYPE).asText()); + boolean isDefaultPresent = chartType.equals(ChartType.LINE) && chartNode.get(Constants.JsonPaths.INTERVAL)!=null; + boolean isRequestContainsInterval = null == request.getRequestDate() ? false : (request.getRequestDate().getInterval()!=null && !request.getRequestDate().getInterval().isEmpty()) ; + String interval = isRequestContainsInterval? request.getRequestDate().getInterval(): (isDefaultPresent ? chartNode.get(Constants.JsonPaths.INTERVAL).asText():""); + + executeConfiguredQueries(chartNode, aggrObjectNode, nodes, request, interval); + request.setChartNode(chartNode); + IResponseHandler responseHandler = responseHandlerFactory.getInstance(chartType); + AggregateDto aggregateDto = new AggregateDto(); + if(aggrObjectNode.fields().hasNext()){ + aggregateDto = responseHandler.translate(request, aggrObjectNode); + } + + if(insightsConfig != null && StringUtils.isNotBlank(insightsConfig.getInsightInterval())) { + continueWithInsight = getInsightsDate(request, insightsConfig.getInsightInterval()); + if(continueWithInsight) { + String insightVisualizationCode = insightPrefix + request.getVisualizationCode(); + request.setVisualizationCode(insightVisualizationCode); + executeConfiguredQueries(chartNode, insightAggrObjectNode, insightNodes, request, interval); + request.setChartNode(chartNode); + responseHandler = responseHandlerFactory.getInstance(chartType); + if(insightAggrObjectNode.fields().hasNext()){ + responseHandler.translate(request, insightAggrObjectNode); + } + InsightsHandler insightsHandler = insightsHandlerFactory.getInstance(chartType); + aggregateDto = insightsHandler.getInsights(aggregateDto, request.getVisualizationCode(), request.getModuleLevel(), insightsConfig); + } + } + + return aggregateDto; + } + + private void executeConfiguredQueries(ObjectNode chartNode, ObjectNode aggrObjectNode, ObjectNode nodes, AggregateRequestDto request, String interval) { + preHandle(request, chartNode, mdmsApiMappings); + + ArrayNode queries = (ArrayNode) chartNode.get(Constants.JsonPaths.QUERIES); + queries.forEach(query -> { + String module = query.get(Constants.JsonPaths.MODULE).asText(); + if(request.getModuleLevel().equals(Constants.Modules.HOME_REVENUE) || + request.getModuleLevel().equals(Constants.Modules.HOME_SERVICES) || + query.get(Constants.JsonPaths.MODULE).asText().equals(Constants.Modules.COMMON) || + request.getModuleLevel().equals(module)) { + + String indexName = query.get(Constants.JsonPaths.INDEX_NAME).asText(); + ObjectNode objectNode = queryService.getChartConfigurationQuery(request, query, indexName, interval); + try { + JsonNode aggrNode = restService.search(indexName,objectNode.toString()); + if(nodes.has(indexName)) { + indexName = indexName + "_1"; + } + nodes.set(indexName,aggrNode.get(Constants.JsonPaths.AGGREGATIONS)); + }catch (Exception e) { + logger.error("Encountered an Exception while Executing the Query : " + e.getMessage()); + } + aggrObjectNode.set(Constants.JsonPaths.AGGREGATIONS, nodes); + + } + }); + } + + private Boolean getInsightsDate(AggregateRequestDto request, String insightInterval) { + Long daysBetween = daysBetween(Long.parseLong(request.getRequestDate().getStartDate()), + Long.parseLong(request.getRequestDate().getEndDate())); + if(insightInterval.equals(Constants.Interval.month.toString()) && daysBetween > 32) { + return Boolean.FALSE; + } + if(insightInterval.equals(Constants.Interval.week.toString()) && daysBetween > 8) { + return Boolean.FALSE; + } + if(insightInterval.equals(Constants.Interval.year.toString()) && daysBetween > 366) { + return Boolean.FALSE; + } + Calendar startCal = Calendar.getInstance(); + Calendar endCal = Calendar.getInstance(); + startCal.setTime(new Date(Long.parseLong(request.getRequestDate().getStartDate()))); + endCal.setTime(new Date(Long.parseLong(request.getRequestDate().getEndDate()))); + if(insightInterval.equals(Constants.Interval.month.toString())) { + startCal.add(Calendar.MONTH, -1); + endCal.add(Calendar.MONTH, -1); + } else if(insightInterval.equals(Constants.Interval.week.toString())) { + startCal.add(Calendar.WEEK_OF_YEAR, -1); + endCal.add(Calendar.WEEK_OF_YEAR, -1); + } else if(StringUtils.isBlank(insightInterval) || insightInterval.equals(Constants.Interval.year.toString())) { + startCal.add(Calendar.YEAR, -1); + endCal.add(Calendar.YEAR, -1); + } + request.getRequestDate().setStartDate(String.valueOf(startCal.getTimeInMillis())); + request.getRequestDate().setEndDate(String.valueOf(endCal.getTimeInMillis())); + return Boolean.TRUE; + } + + public long daysBetween(Long start, Long end) { + return TimeUnit.MILLISECONDS.toDays(Math.abs(end - start)); + } + + + + @Override + public List getHeaderData(CummulativeDataRequestDto requestDto, List roles) throws AINException { + // TODO Auto-generated method stub + return null; + } + + + +} diff --git a/dashboard-analytics/src/main/java/com/tarento/analytics/query/model/Aggregation.java b/dashboard-analytics/src/main/java/com/tarento/analytics/query/model/Aggregation.java new file mode 100644 index 000000000..b7f4c8a28 --- /dev/null +++ b/dashboard-analytics/src/main/java/com/tarento/analytics/query/model/Aggregation.java @@ -0,0 +1,5 @@ +package com.tarento.analytics.query.model; + +public class Aggregation { + +} diff --git a/dashboard-analytics/src/main/java/com/tarento/analytics/query/model/Query.java b/dashboard-analytics/src/main/java/com/tarento/analytics/query/model/Query.java new file mode 100644 index 000000000..d48b10b25 --- /dev/null +++ b/dashboard-analytics/src/main/java/com/tarento/analytics/query/model/Query.java @@ -0,0 +1,28 @@ +package com.tarento.analytics.query.model; + +import com.fasterxml.jackson.annotation.JsonProperty; + +public class Query { + + @JsonProperty("x_axis") + public XAxis xAxis; + + @JsonProperty("y_axis") + public YAxis yAxis; + + public XAxis getxAxis() { + return xAxis; + } + + public void setxAxis(XAxis xAxis) { + this.xAxis = xAxis; + } + + public YAxis getyAxis() { + return yAxis; + } + + public void setyAxis(YAxis yAxis) { + this.yAxis = yAxis; + } +} diff --git a/dashboard-analytics/src/main/java/com/tarento/analytics/query/model/XAxis.java b/dashboard-analytics/src/main/java/com/tarento/analytics/query/model/XAxis.java new file mode 100644 index 000000000..e7b5a9bed --- /dev/null +++ b/dashboard-analytics/src/main/java/com/tarento/analytics/query/model/XAxis.java @@ -0,0 +1,18 @@ +package com.tarento.analytics.query.model; + +import java.util.Map; + +public class XAxis { + + public Map aggregation; + + public Map getAggregation() { + return aggregation; + } + + public void setAggregation(Map aggregation) { + this.aggregation = aggregation; + } + + +} diff --git a/dashboard-analytics/src/main/java/com/tarento/analytics/query/model/YAxis.java b/dashboard-analytics/src/main/java/com/tarento/analytics/query/model/YAxis.java new file mode 100644 index 000000000..e44af59a1 --- /dev/null +++ b/dashboard-analytics/src/main/java/com/tarento/analytics/query/model/YAxis.java @@ -0,0 +1,17 @@ +package com.tarento.analytics.query.model; + +import java.util.Map; + +public class YAxis { + + Map aggregation; + + public Map getAggregation() { + return aggregation; + } + + public void setAggregation(Map aggregation) { + this.aggregation = aggregation; + } + +} diff --git a/dashboard-analytics/src/main/java/com/tarento/analytics/repository/ElasticSearchRepository.java b/dashboard-analytics/src/main/java/com/tarento/analytics/repository/ElasticSearchRepository.java new file mode 100644 index 000000000..4da1f8de9 --- /dev/null +++ b/dashboard-analytics/src/main/java/com/tarento/analytics/repository/ElasticSearchRepository.java @@ -0,0 +1,84 @@ +package com.tarento.analytics.repository; + +import java.util.Map; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.http.HttpEntity; +import org.springframework.http.HttpHeaders; +import org.springframework.http.HttpMethod; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.stereotype.Service; +import org.springframework.web.client.HttpClientErrorException; +import org.springframework.web.client.HttpServerErrorException; +import org.springframework.web.client.RestTemplate; + +import com.tarento.analytics.dto.MappingDto; +import com.tarento.analytics.model.Transaction; + +/** + * This Repository Class is used to perform the transactions of storing the data into the Elastic Search Repository + * @author Darshan Nagesh + * + */ +@Service +public class ElasticSearchRepository { + + public static final Logger LOGGER = LoggerFactory.getLogger(ElasticSearchRepository.class); + + private final RestTemplate restTemplate; + + public ElasticSearchRepository(RestTemplate restTemplate) { + this.restTemplate = restTemplate; + } + + /** + * Based on the Transaction Index Data Obtained and the URL with Headers, this method will put the Data obtained on the + * Elastic Search Database and returns the response in the form of Positive or Negative outcome (True Or False) + * @param transactionIndex + * @param url + * @param headers + * @return + */ + public Boolean saveTransaction(Transaction transaction, String url, HttpHeaders headers) { + ResponseEntity map = null; + try { + map = restTemplate.exchange(url, HttpMethod.PUT, + new HttpEntity<>(transaction, headers), Map.class); + } catch (final HttpClientErrorException httpClientErrorException) { + LOGGER.error("Error : " + httpClientErrorException); + } catch (HttpServerErrorException httpServerErrorException) { + LOGGER.error("Error : " + httpServerErrorException); + } catch (Exception e) { + LOGGER.error("Error : " + e); + } + if (map != null && map.getStatusCode() != null && (map.getStatusCode() == HttpStatus.OK) || (map.getStatusCode() == HttpStatus.CREATED)) { + return true; + } + return false; + } + + + + // create index + public Boolean createTransactionIndex(MappingDto dto, String url, HttpHeaders headers){ + ResponseEntity map = null; + try { + LOGGER.info("Each Channel Content : " + dto.toString()); + map = restTemplate.exchange(url, HttpMethod.PUT, + new HttpEntity<>(dto, headers), Map.class); + } catch (final HttpClientErrorException httpClientErrorException) { + LOGGER.error("Error : " + httpClientErrorException); + } catch (HttpServerErrorException httpServerErrorException) { + LOGGER.error("Error : " + httpServerErrorException); + } catch (Exception e) { + LOGGER.error("Error : " + e); + } + if (map != null && map.getStatusCode() != null && (map.getStatusCode() == HttpStatus.OK) || (map.getStatusCode() == HttpStatus.CREATED)) { + return true; + } + return false; + } + +} diff --git a/dashboard-analytics/src/main/java/com/tarento/analytics/service/AmazonS3ClientService.java b/dashboard-analytics/src/main/java/com/tarento/analytics/service/AmazonS3ClientService.java new file mode 100644 index 000000000..1ebda9f0d --- /dev/null +++ b/dashboard-analytics/src/main/java/com/tarento/analytics/service/AmazonS3ClientService.java @@ -0,0 +1,10 @@ +package com.tarento.analytics.service; + +import org.springframework.web.multipart.MultipartFile; + +public interface AmazonS3ClientService +{ + String uploadFileToS3Bucket(MultipartFile multipartFile, boolean enablePublicReadAccess); + + void deleteFileFromS3Bucket(String fileName); +} diff --git a/dashboard-analytics/src/main/java/com/tarento/analytics/service/MetadataService.java b/dashboard-analytics/src/main/java/com/tarento/analytics/service/MetadataService.java new file mode 100644 index 000000000..5a2c1c530 --- /dev/null +++ b/dashboard-analytics/src/main/java/com/tarento/analytics/service/MetadataService.java @@ -0,0 +1,16 @@ +package com.tarento.analytics.service; + +import java.io.IOException; +import java.util.List; + +import org.json.JSONArray; + +import com.tarento.analytics.dto.RoleDto; +import com.tarento.analytics.exception.AINException; + +public interface MetadataService { + + public Object getDashboardConfiguration(String dashboardId, String catagory, List roleIds) throws AINException, IOException; + public JSONArray getTargetDistrict() throws Exception ; + +} diff --git a/dashboard-analytics/src/main/java/com/tarento/analytics/service/QueryService.java b/dashboard-analytics/src/main/java/com/tarento/analytics/service/QueryService.java new file mode 100644 index 000000000..bc97d7582 --- /dev/null +++ b/dashboard-analytics/src/main/java/com/tarento/analytics/service/QueryService.java @@ -0,0 +1,30 @@ +package com.tarento.analytics.service; + +import java.io.IOException; + +import org.elasticsearch.search.aggregations.Aggregations; + +import com.fasterxml.jackson.core.JsonParseException; +import com.fasterxml.jackson.databind.JsonMappingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.tarento.analytics.dto.AggregateRequestDto; +import com.tarento.analytics.dto.AggregateRequestDto; +import com.tarento.analytics.exception.AINException; + +public interface QueryService { +/* + List getAggregateData(AggregateRequestDto aggregateDto, String orgId) throws AINException; +*/ + public static final String API_CONFIG_JSON = "ChartApiConfig.json"; + public static final String DATE_SOURCE_FIELD = "dateSourceField"; + public static final String AGG_QUERY_JSON = "aggregationQueryJson"; + public static final String INDEX_NAME = "indexName" ; + public static final String DOCUMENT_TYPE = "documentType"; +/* Aggregations getAggregateData(AggregateRequestDto aggregateDto, String orgId) throws AINException; + Aggregations getAggregateDataV2(AggregateRequestDtoV2 aggregateDto, String orgId) throws AINException, JsonParseException, JsonMappingException, IOException; + */ + ObjectNode getChartConfigurationQuery(AggregateRequestDto req, JsonNode query, String indexName, String interval); + + +} diff --git a/dashboard-analytics/src/main/java/com/tarento/analytics/service/impl/AmazonS3ClientServiceImpl.java b/dashboard-analytics/src/main/java/com/tarento/analytics/service/impl/AmazonS3ClientServiceImpl.java new file mode 100644 index 000000000..651f8aa9b --- /dev/null +++ b/dashboard-analytics/src/main/java/com/tarento/analytics/service/impl/AmazonS3ClientServiceImpl.java @@ -0,0 +1,85 @@ +package com.tarento.analytics.service.impl; + +import com.amazonaws.AmazonServiceException; +import com.amazonaws.auth.AWSCredentialsProvider; +import com.amazonaws.regions.Region; +import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.AmazonS3ClientBuilder; +import com.amazonaws.services.s3.model.CannedAccessControlList; +import com.amazonaws.services.s3.model.DeleteObjectRequest; +import com.amazonaws.services.s3.model.PutObjectRequest; +import com.tarento.analytics.model.AmazonS3Config; +import com.tarento.analytics.service.AmazonS3ClientService; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.scheduling.annotation.Async; +import org.springframework.stereotype.Component; +import org.springframework.web.multipart.MultipartFile; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.util.Date; + +@Component +public class AmazonS3ClientServiceImpl implements AmazonS3ClientService +{ + @Autowired + private AmazonS3Config amazonS3Config; + private String awsS3AudioBucket; + private AmazonS3 amazonS3; + private static final Logger logger = LoggerFactory.getLogger(AmazonS3ClientServiceImpl.class); + + @Autowired + public AmazonS3ClientServiceImpl(Region awsRegion, AWSCredentialsProvider awsCredentialsProvider, String awsS3AudioBucket) + { + this.amazonS3 = AmazonS3ClientBuilder.standard() + .withCredentials(awsCredentialsProvider) + .withRegion(awsRegion.getName()).build(); + this.awsS3AudioBucket = awsS3AudioBucket; + } + + @Async + public String uploadFileToS3Bucket(MultipartFile multipartFile, boolean enablePublicReadAccess) + { + String fileName = multipartFile.getOriginalFilename(); + long time = new Date().getTime(); + + String imageURL = ""; + try { + //creating the file in the server (temporarily) + File file = new File(time+"-"+fileName); + FileOutputStream fos = new FileOutputStream(file); + fos.write(multipartFile.getBytes()); + fos.close(); + + PutObjectRequest putObjectRequest = new PutObjectRequest(this.awsS3AudioBucket, fileName, file); + + if (enablePublicReadAccess) { + putObjectRequest.withCannedAcl(CannedAccessControlList.PublicRead); + } + this.amazonS3.putObject(putObjectRequest); + + imageURL = String.valueOf(amazonS3.getUrl( + amazonS3Config.getAWSS3AudioBucket(), //The S3 Bucket To Upload To + file.getName())); + //removing the file created in the server + file.delete(); + } catch (IOException | AmazonServiceException ex) { + logger.error("error [" + ex.getMessage() + "] occurred while uploading [" + fileName + "] "); + } + return imageURL; + + } + + @Async + public void deleteFileFromS3Bucket(String fileName) + { + try { + amazonS3.deleteObject(new DeleteObjectRequest(awsS3AudioBucket, fileName)); + } catch (AmazonServiceException ex) { + logger.error("error [" + ex.getMessage() + "] occurred while removing [" + fileName + "] "); + } + } +} \ No newline at end of file diff --git a/dashboard-analytics/src/main/java/com/tarento/analytics/service/impl/MetadataServiceImpl.java b/dashboard-analytics/src/main/java/com/tarento/analytics/service/impl/MetadataServiceImpl.java new file mode 100644 index 000000000..d248cad44 --- /dev/null +++ b/dashboard-analytics/src/main/java/com/tarento/analytics/service/impl/MetadataServiceImpl.java @@ -0,0 +1,216 @@ +package com.tarento.analytics.service.impl; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileReader; +import java.io.IOException; +import java.net.URI; +import java.util.*; + +import com.tarento.analytics.constant.Constants; +import org.json.JSONArray; +import org.json.JSONObject; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.http.HttpEntity; +import org.springframework.http.HttpHeaders; +import org.springframework.http.HttpMethod; +import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; +import org.springframework.stereotype.Service; +import org.springframework.web.client.RestTemplate; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.JsonNodeFactory; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.google.gson.Gson; +import com.tarento.analytics.ConfigurationLoader; +import com.tarento.analytics.dto.RoleDto; +import com.tarento.analytics.dto.Tenants; +import com.tarento.analytics.exception.AINException; +import com.tarento.analytics.service.MetadataService; + +@Service("metadataService") +public class MetadataServiceImpl implements MetadataService { + + public static final Logger logger = LoggerFactory.getLogger(MetadataServiceImpl.class); + + @Autowired + private ConfigurationLoader configurationLoader; + + @Autowired + private ObjectMapper objectMapper; + + @Autowired + private RestTemplate restTemplate; + + + @Value("${egov.mdms-service.target.url}") + private String mdmsServiceTargetUrl; + + + @Override + public ArrayNode getDashboardConfiguration(String dashboardId, String catagory, List roleIds) throws AINException, IOException { + + Calendar cal = Calendar.getInstance(); + cal.set(cal.getWeekYear()-1, Calendar.APRIL, 1); + Date startDate = cal.getTime(); + Date endDate = new Date(); + + String fyInfo = "From " + Constants.DASHBOARD_DATE_FORMAT.format(startDate) + " to " + Constants.DASHBOARD_DATE_FORMAT.format(endDate); + + + ObjectNode dashBoardNode = configurationLoader.get(ConfigurationLoader.MASTER_DASHBOARD_CONFIG); + ArrayNode dasboardNodes = (ArrayNode) dashBoardNode.findValue(Constants.DashBoardConfig.DASHBOARDS); + + ObjectNode roleMappingNode = configurationLoader.get(ConfigurationLoader.ROLE_DASHBOARD_CONFIG); + ArrayNode rolesArray = (ArrayNode) roleMappingNode.findValue(Constants.DashBoardConfig.ROLES); + ArrayNode dbArray = JsonNodeFactory.instance.arrayNode(); + for(JsonNode role: rolesArray){ + logger.info("role name: " + role.get("roleName")); + logger.info("role ID: " + role.get("roleId")); + String roleId = role.get("roleId").asText(); + + //Object roleId = roleIds.stream().filter(x -> role.get(Constants.DashBoardConfig.ROLE_ID).asLong() == (x.getId())).findAny().orElse(null); + if (null != roleId) { + ArrayNode visArray = JsonNodeFactory.instance.arrayNode(); + for(JsonNode db : role.get(Constants.DashBoardConfig.DASHBOARDS)){ + ObjectNode copyDashboard = objectMapper.createObjectNode(); + + JsonNode name = JsonNodeFactory.instance.textNode(""); + JsonNode id = JsonNodeFactory.instance.textNode(""); + JsonNode title = JsonNodeFactory.instance.textNode(fyInfo); + if (db.get(Constants.DashBoardConfig.ID).asText().equalsIgnoreCase(dashboardId)) { + //dasboardNodes.forEach(dbNode -> { + for(JsonNode dbNode : dasboardNodes){ + if (dbNode.get(Constants.DashBoardConfig.ID).asText().equalsIgnoreCase(dashboardId)) { + logger.info("dbNode: " + dbNode); + name = dbNode.get(Constants.DashBoardConfig.NAME); + id = dbNode.get(Constants.DashBoardConfig.ID); + + if (catagory != null) { + dbNode.get(Constants.DashBoardConfig.VISUALISATIONS).forEach(visual -> { + if (visual.get(Constants.DashBoardConfig.NAME).asText().equalsIgnoreCase(catagory)) + visArray.add(visual); + }); + } else { + dbNode.get(Constants.DashBoardConfig.VISUALISATIONS).forEach(visual -> { + visArray.add(visual); + }); + } + } + copyDashboard.set(Constants.DashBoardConfig.NAME, name); + copyDashboard.set(Constants.DashBoardConfig.ID, id); + //add TITLE with varible dynamically + copyDashboard.set(Constants.DashBoardConfig.TITLE, title); + + copyDashboard.set(Constants.DashBoardConfig.VISUALISATIONS, visArray); + copyDashboard.set("roleId", role.get("roleId")); + copyDashboard.set("roleName", role.get("roleName")); + + }//); + dbArray.add(copyDashboard); + } + } + + } + } + + return dbArray; + } + +/* @Override + public ArrayNode getDashboardConfiguration(String dashboardId, String catagory, List roleIds) throws AINException, IOException { + + ObjectNode configNode = configurationLoader.get(ConfigurationLoader.ROLE_DASHBOARD_CONFIG); + ArrayNode rolesArray = (ArrayNode) configNode.findValue("roles"); + ArrayNode dbArray = JsonNodeFactory.instance.arrayNode(); + + rolesArray.forEach(role -> { + Object roleId = roleIds.stream().filter(x -> role.get("roleId").asLong() == (x.getId())).findAny().orElse(null); + System.out.println("roleId = "+roleId); + + if (null != roleId) { + role.get("dashboards").forEach(dashboard -> { + ObjectNode copyDashboard = dashboard.deepCopy(); + ArrayNode visArray = JsonNodeFactory.instance.arrayNode(); + if(catagory != null) { + copyDashboard.get("visualizations").forEach(visual ->{ + if(visual.get("name").asText().equalsIgnoreCase(catagory)){ + visArray.add(visual); + } + }); + copyDashboard.set("visualizations", visArray); + } + if(dashboard.get("id").asText().equalsIgnoreCase(dashboardId)){ + dbArray.add(copyDashboard); + } + }); + } + }); + + //List dbs = objectMapper.readValue(dbArray.toString(), new TypeReference>() {}); + return dbArray; + }*/ + + public JSONArray getTargetDistrict() throws Exception { + final String baseUrl = mdmsServiceTargetUrl; + URI uri = new URI(baseUrl); + HttpHeaders headers = new HttpHeaders(); + headers.setContentType(MediaType.APPLICATION_JSON); + HttpEntity requestEntity = new HttpEntity<>("{}", headers); + ResponseEntity response = restTemplate.exchange(uri, HttpMethod.POST, requestEntity, String.class); + String targetdistrict = response.getBody(); + ObjectMapper objectMapper = new ObjectMapper(); + JsonNode jsonNode = objectMapper.readTree(targetdistrict); + ArrayNode array = (ArrayNode) jsonNode.at("/MdmsRes/tenant/tenants"); + + File getFile = new File( + System.getProperty("user.dir") + System.getProperty("file.separator") + "data/tenants.json"); + BufferedReader br = new BufferedReader(new FileReader(getFile)); + Tenants sample = new Gson().fromJson(br, Tenants.class); + JSONArray jsonArray = new JSONArray(); + Map> mapDistrictUlb = new HashMap(); + Map DistrictMap = new HashMap(); + for (int i = 0; i < sample.getTenants().size(); i++) { + String ulbCode = sample.getTenants().get(i).getCode(); + String ulbName = sample.getTenants().get(i).getName(); + String districtName = sample.getTenants().get(i).getCity().getDistrictName(); + String districtCode = sample.getTenants().get(i).getCity().getDistrictCode(); + Map mapUlb = new HashMap(); + Map mapDist = new LinkedHashMap(); + Map mapDistAll = new LinkedHashMap(); + mapDist.put("District Name", districtName); + mapDist.put("District Code", districtCode); + mapUlb.put("Ulb Name", ulbName); + mapUlb.put("tenantId", ulbCode); + mapDistAll.put(districtCode, new JSONObject(mapDist)); + DistrictMap.put(districtCode.toString(), mapDist); + if (mapDistrictUlb.containsKey(districtCode.toString())) { + mapDistrictUlb.get(districtCode.toString()).add(mapUlb); + } else { + List lst = new ArrayList(); + lst.add(mapUlb); + mapDistrictUlb.put(districtCode.toString(), lst); + } + } + for (Map.Entry entry1 : DistrictMap.entrySet()) { + JSONObject getdistrictJson = new JSONObject(); + for (Map.Entry> entry2 : mapDistrictUlb.entrySet()) { + if (entry1.getKey().equals(entry2.getKey())) { + JSONObject getUlbtoDistrict = new JSONObject(); + getdistrictJson.put(entry1.getKey(), entry1.getValue()); + String json = new Gson().toJson(entry1.getValue(), LinkedHashMap.class); + getUlbtoDistrict.put(entry1.getKey(), new JSONObject(json).accumulate("Ulb", entry2.getValue())); + jsonArray.put(getUlbtoDistrict); + } + } + } + return jsonArray; + } + +} diff --git a/dashboard-analytics/src/main/java/com/tarento/analytics/service/impl/QueryServiceImpl.java b/dashboard-analytics/src/main/java/com/tarento/analytics/service/impl/QueryServiceImpl.java new file mode 100644 index 000000000..bd3429ebd --- /dev/null +++ b/dashboard-analytics/src/main/java/com/tarento/analytics/service/impl/QueryServiceImpl.java @@ -0,0 +1,460 @@ +package com.tarento.analytics.service.impl; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; + +import org.elasticsearch.action.search.MultiSearchResponse; +import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.search.aggregations.Aggregation; +import org.elasticsearch.search.aggregations.Aggregations; +import org.elasticsearch.search.aggregations.bucket.terms.ParsedLongTerms; +import org.elasticsearch.search.aggregations.bucket.terms.Terms; +import org.elasticsearch.search.aggregations.metrics.avg.ParsedAvg; +import org.elasticsearch.search.aggregations.metrics.sum.ParsedSum; +import org.elasticsearch.search.aggregations.metrics.valuecount.ParsedValueCount; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +import com.fasterxml.jackson.core.JsonParseException; +import com.fasterxml.jackson.databind.JsonMappingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.tarento.analytics.ConfigurationLoader; +import com.tarento.analytics.constant.Constants; +import com.tarento.analytics.dao.ElasticSearchDao; +import com.tarento.analytics.dto.AggregateRequestDto; +import com.tarento.analytics.dto.AggregateRequestDto; +import com.tarento.analytics.enums.ChartType; +import com.tarento.analytics.exception.AINException; +import com.tarento.analytics.model.ElasticSearchDictator; +import com.tarento.analytics.model.KeyData; +import com.tarento.analytics.model.Query; +import com.tarento.analytics.model.ServiceQuery; +import com.tarento.analytics.service.QueryService; +import com.tarento.analytics.utils.ElasticProperties; + +@Component +public class QueryServiceImpl implements QueryService { + + public static final Logger logger = LoggerFactory.getLogger(QueryServiceImpl.class); +/* + @Autowired + private MetadataDao metaDataDao;*/ + + @Autowired + private ElasticSearchDao elasticSearchDao; + +/* + @Autowired + private DecoratorService decoratorService; + +*/ + @Autowired + private ConfigurationLoader configurationLoader; + + private static final Map WeekDayMap = createMap(); + + private static Map createMap() { + Map result = new HashMap(); + result.put(1, "SUN"); + result.put(2, "MON"); + result.put(3, "TUE"); + result.put(4, "WED"); + result.put(5, "THU"); + result.put(6, "FRI"); + result.put(7, "SAT"); + + return Collections.unmodifiableMap(result); + } + +/* + @Override + public Aggregations getAggregateData(AggregateRequestDto aggregateDto, String orgId) throws AINException { + + //public List getAggregateData(AggregateRequestDto aggregateDto, String orgId) throws AINException { + Aggregations aggregations = null; + ElasticSearchDictator dictator = null; + ServiceQuery elasticServiceQuery = metaDataDao.getServiceQuery(Long.parseLong(orgId), + aggregateDto.getServiceApi()); + + ObjectMapper mapper = new ObjectMapper(); + */ +/*Map resonseMap = null;*//* + + try { + + //Query Builder + Query queryJson = mapper.readValue(elasticServiceQuery.getQueryJson(), Query.class); + // Set index name and document Type for elastic + Map queryMap = queryJson.getAggregation(); + + //Label Mapping + Map labelMap = new HashMap<>(); + getAggregateLabelRecursively(queryMap,labelMap); + + dictator = elasticSearchDao.createSearchDictator(aggregateDto, elasticServiceQuery.getIndexName(), + elasticServiceQuery.getDocumentType(), queryJson.getDateFilterField()); + dictator.setQueryAggregationMap(queryMap); + SearchRequest searchRequest = elasticSearchDao.buildElasticSearchQuery(dictator); + List searchRequestList = new ArrayList<>(); + searchRequestList.add(searchRequest); + MultiSearchResponse response = elasticSearchDao.executeMultiSearchRequest(searchRequestList, Boolean.TRUE); + SearchResponse searchResponse = response.getResponses()[0].getResponse(); + aggregations = searchResponse.getAggregations(); + */ +/*resonseMap= translateResponse(response, aggregateDto.getChartType(),aggregateDto.getInterval(), labelMap); + decoratorService.getChartData(aggregateDto, chartFormat, chartType, serviceApi, chartCode)*//* + + } catch (Exception e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + return aggregations; + } + + @Override + public Aggregations getAggregateDataV2(AggregateRequestDtoV2 aggregateDto, String orgId) throws AINException, JsonParseException, JsonMappingException, IOException { + Boolean primaryOrNot = Boolean.TRUE; + ServiceQuery elasticServiceQuery = new ServiceQuery(); + String visualizationCode = aggregateDto.getVisualizationCode(); + ObjectNode configNode = configurationLoader.get(API_CONFIG_JSON); + elasticServiceQuery.setIndexName(configNode.get(visualizationCode).get(INDEX_NAME).asText()); + elasticServiceQuery.setDocumentType(configNode.get(visualizationCode).get(DOCUMENT_TYPE).asText()); + elasticServiceQuery.setQueryJson(configNode.get(visualizationCode).get(AGG_QUERY_JSON).asText()); + if(elasticServiceQuery.getIndexName().equals("dss-col-v1")) { + primaryOrNot = Boolean.TRUE; + } else { + primaryOrNot = Boolean.FALSE; + } + ObjectMapper mapper = new ObjectMapper(); + Query queryJson = mapper.readValue(elasticServiceQuery.getQueryJson(), Query.class); + + // Set index name and document Type for elastic + Map queryMap = queryJson.getAggregation(); + + //Label Mapping + Map labelMap = new HashMap<>(); + getAggregateLabelRecursively(queryMap,labelMap); + + Aggregations aggregations = null; + ElasticSearchDictator dictator = null; + + try { + dictator = elasticSearchDao.createSearchDictatorV2(aggregateDto, elasticServiceQuery.getIndexName(), + elasticServiceQuery.getDocumentType(), queryJson.getDateFilterField()); + dictator.setQueryAggregationMap(queryMap); + SearchRequest searchRequest = elasticSearchDao.buildElasticSearchQuery(dictator); + List searchRequestList = new ArrayList<>(); + searchRequestList.add(searchRequest); + MultiSearchResponse response = elasticSearchDao.executeMultiSearchRequest(searchRequestList, primaryOrNot); + SearchResponse searchResponse = response.getResponses()[0].getResponse(); + aggregations = searchResponse.getAggregations(); + } catch (Exception e) { + logger.error("Encountered an error while getting the Aggregated Data : " + e.getMessage()); + } + return aggregations; + } +*/ + +/* private Map translateResponse(MultiSearchResponse response, ChartType chartType, String interval, Map labelMap) { + SearchResponse searchResponse = response.getResponses()[0].getResponse(); + List aggregations = searchResponse.getAggregations().asList(); + Map responseMap = new HashMap<>(); + + if (chartType.equals(ChartType.BAR) || chartType.equals(ChartType.HORIZONTALBAR) + || chartType.equals(ChartType.PIE) || chartType.equals(ChartType.DOUGHNUT)) { + + if (aggregations.get(0) instanceof ParsedLongTerms) { + responseMap = parseParseLongTerms((ParsedLongTerms) aggregations.get(0), chartType, labelMap); + } + } + + return responseMap; + }*/ + + @SuppressWarnings("unchecked") + void getAggregateLabelRecursively(Map queryMap, Map labelMap ){ + try { + if(queryMap.containsKey(ElasticProperties.Query.AGGREGATION_CONDITION.toLowerCase())){ + + Map valueMap =(HashMap)queryMap.get(ElasticProperties.Query.AGGREGATION_CONDITION.toLowerCase()); + getAggregateLabelRecursively(valueMap, labelMap); + } + for (Map.Entry itrQuery : queryMap.entrySet()) { + if(itrQuery.getKey().equals(ElasticProperties.Query.AGGREGATION_CONDITION.toLowerCase())){ + continue; + } + Map propertiesMap = (HashMap)itrQuery.getValue(); + labelMap.put(itrQuery.getKey(), propertiesMap.get(ElasticProperties.Query.LABEL.toLowerCase()).toString()); + } + } catch (Exception e) { + + } + } + /* + * private Map + * parseDateHistogramForMultiLine(List aggregations, ChartType + * chartType, String interval) { Map responseMap = new + * HashMap<>(); + * + * for (Histogram.Bucket bucket : ((ParsedDateHistogram) + * aggregations.get(0)).getBuckets()) { List subAggregations = + * bucket.getAggregations().asList(); if (subAggregations.get(0) instanceof + * ParsedLongTerms) { parseParseLongTermsOnInterval(bucket, chartType, + * interval, responseMap); } + * + * } + * + * return responseMap; + * + * } + */ + + /* + * private Map parseNested(List aggregations, + * ChartType chartType, String interval) { + * + * Map responseMap = new HashMap<>(); for (Aggregation + * aggregationData : ((ParsedNested) aggregations.get(0)).getAggregations()) + * { if (aggregationData instanceof ParsedLongTerms) { responseMap = + * parseParseLongTerms((ParsedLongTerms) aggregationData, chartType, + * interval); } + * + * } return responseMap; + * + * } + */ + + /* + * private Map parseDateHistogram(List + * aggregations, ChartType chartType, String interval) { + * + * Map responseMap = new HashMap<>(); if + * (interval.equals(ElasticSearchConstants.DAY_OF_WEEK)) { Map dayWiseObjectMap = new HashMap() { { put(1, + * null); put(2, null); put(3, null); put(4, null); put(5, null); put(6, + * null); put(7, null); + * + * } }; + * + * for (Histogram.Bucket bucket : ((ParsedDateHistogram) + * aggregations.get(0)).getBuckets()) { Calendar cal = + * Calendar.getInstance(); + * cal.setTimeInMillis(Long.parseLong(bucket.getKeyAsString())); int + * dayofWeek = cal.get(Calendar.DAY_OF_WEEK); + * + * Object val = getAggregationValue(bucket); if + * (dayWiseObjectMap.containsKey(dayofWeek)) { Object dayWiseCount = + * dayWiseObjectMap.get(dayofWeek); if (val instanceof Double) { Double + * doubleValue = 0.0; if (dayWiseCount == null) { doubleValue = 0.0; } else + * { doubleValue = (Double) dayWiseCount; } doubleValue += + * getFormattedDouble((Double) val); dayWiseCount = doubleValue; } else if + * (val instanceof Long) { Long longValue = 0L; if (dayWiseCount == null) { + * longValue = 0L; } else { longValue = (Long) dayWiseCount; } longValue += + * (Long) val; dayWiseCount = longValue; } dayWiseObjectMap.put(dayofWeek, + * dayWiseCount); } else { if (val instanceof Double) { Double doubleValue = + * (Double) val; doubleValue += getFormattedDouble((Double) val); val = + * doubleValue; } dayWiseObjectMap.put(dayofWeek, val); } } for + * (Map.Entry entry : dayWiseObjectMap.entrySet()) { + * + * responseMap.put(WeekDayMap.get(entry.getKey()), (entry.getValue())); } } + * else { for (Histogram.Bucket bucket : ((ParsedDateHistogram) + * aggregations.get(0)).getBuckets()) { List subAggregations = + * bucket.getAggregations().asList(); if (subAggregations.get(0) instanceof + * ParsedLongTerms) { + * + * + * responseMap = parseParseLongTermsOnInterval(bucket, chartType, interval); + * } else { responseMap = parseDateHistogramBasedOnInterval(bucket, + * chartType, interval); } + * + * } } return responseMap; + * + * } + */ + /* + * private void parseParseLongTermsOnInterval(Bucket buckets, ChartType + * chartType, String interval, Map responseMap) { Calendar + * cal = Calendar.getInstance(); + * cal.setTimeInMillis(Long.parseLong(buckets.getKeyAsString())); String key + * = null; if (interval.equals(ElasticSearchConstants.DAY)) { int day = + * cal.get(Calendar.DAY_OF_MONTH); int month = cal.get(Calendar.MONTH) + 1; + * + * key = day + "/" + month; } else if + * (interval.equals(ElasticSearchConstants.HOUR)) { int hour = + * cal.get(Calendar.HOUR_OF_DAY); String suffix = "AM"; if + * (cal.get(Calendar.AM_PM) > 0) suffix = "PM"; + * + * key = String.valueOf(hour + 1) + suffix; + * + * } Map innerResponseMap = new HashMap<>(); + * + * for (Terms.Bucket bucket : ((ParsedLongTerms) buckets).getBuckets()) { + * + * Map valueMap = bucket.getAggregations().getAsMap(); + * Object val = null; + * + * for (Map.Entry aggregation : valueMap.entrySet()) { + * if + * (aggregation.getKey().contains(ElasticProperties.Query.SUM.toLowerCase()) + * ) { ParsedSum sum = (ParsedSum) aggregation.getValue(); val = + * sum.getValue(); } else if + * (aggregation.getKey().contains(ElasticProperties.Query.AVG.toLowerCase()) + * ) { ParsedAvg avg = (ParsedAvg) aggregation.getValue(); val = + * avg.getValue(); + * + * } else if + * (aggregation.getKey().contains(ElasticProperties.Query.COUNT.toLowerCase( + * ))) { ParsedValueCount count = (ParsedValueCount) aggregation.getValue(); + * val = count.getValue(); + * + * } + * + * } innerResponseMap.put(bucket.getKeyAsString(), val); } + * responseMap.put(key, innerResponseMap); + * + * } + */ + + /* + * private Map parseDateHistogramBasedOnInterval(Bucket + * bucket, ChartType chartType, String interval) { Map + * responseMap = new HashMap<>(); Calendar cal = Calendar.getInstance(); + * cal.setTimeInMillis(Long.parseLong(bucket.getKeyAsString())); String key + * = null; Object val = null; if + * (interval.equals(ElasticSearchConstants.DAY)) { int day = + * cal.get(Calendar.DAY_OF_MONTH); int month = cal.get(Calendar.MONTH) + 1; + * + * key = day + "/" + month; val = getAggregationValue(bucket); + * + * } else if (interval.equals(ElasticSearchConstants.HOUR)) { int hour = + * cal.get(Calendar.HOUR_OF_DAY); String suffix = "AM"; if + * (cal.get(Calendar.AM_PM) > 0) suffix = "PM"; val = + * getAggregationValue(bucket); + * + * key = String.valueOf(hour + 1) + suffix; + * + * } responseMap.put(key, val); return responseMap; + * + * } + */ + + /* + * private Object getAggregationValue(Bucket bucket) { Map valueMap = bucket.getAggregations().getAsMap(); Object val = + * null; + * + * for (Map.Entry aggregation : valueMap.entrySet()) { + * if + * (aggregation.getKey().contains(ElasticProperties.Query.SUM.toLowerCase()) + * ) { ParsedSum sum = (ParsedSum) aggregation.getValue(); val = + * sum.getValue(); } else if + * (aggregation.getKey().contains(ElasticProperties.Query.AVG.toLowerCase()) + * ) { ParsedAvg avg = (ParsedAvg) aggregation.getValue(); val = + * avg.getValue(); + * + * } else if + * (aggregation.getKey().contains(ElasticProperties.Query.COUNT.toLowerCase( + * ))) { ParsedValueCount count = (ParsedValueCount) aggregation.getValue(); + * val = count.getValue(); + * + * } } return val; } + */ + + private Map parseParseLongTerms(ParsedLongTerms aggregations, ChartType chartType, Map labelMap) { + + Map keyValueMap = new HashMap<>(); + + for (Terms.Bucket bucket : aggregations.getBuckets()) { + KeyData keyData = new KeyData(); + KeyData valueData = new KeyData(); + + String key = bucket.getKey().toString(); + + Map valueMap = bucket.getAggregations().getAsMap(); + Object val = null; + String valueLabel = null; + for (Map.Entry aggregation : valueMap.entrySet()) { + if (aggregation.getKey().contains(ElasticProperties.Query.SUM.toLowerCase())) { + ParsedSum sum = (ParsedSum) aggregation.getValue(); + val = sum.getValue(); + valueLabel = ElasticProperties.Query.SUM.toLowerCase(); + } else if (aggregation.getKey().contains(ElasticProperties.Query.AVG.toLowerCase())) { + ParsedAvg avg = (ParsedAvg) aggregation.getValue(); + val = avg.getValue(); + valueLabel = ElasticProperties.Query.SUM.toLowerCase(); + + } else if (aggregation.getKey().contains(ElasticProperties.Query.COUNT.toLowerCase())) { + ParsedValueCount count = (ParsedValueCount) aggregation.getValue(); + val = count.getValue(); + valueLabel = ElasticProperties.Query.SUM.toLowerCase(); + + } + + } + keyData.setKey(key); + keyData.setLabel(labelMap.get(ElasticProperties.Query.TERM.toLowerCase())); + valueData.setKey(val); + valueData.setLabel(labelMap.get(valueLabel)); + keyValueMap.put(keyData, valueData); + + } + return keyValueMap; + + } + + private Double getFormattedDouble(double val) { + return (Math.round(new Double(val).isInfinite() ? 0.0 : new Double(val) * 100.0) / 100.0); + } + + @Override + public ObjectNode getChartConfigurationQuery(AggregateRequestDto request, JsonNode query, String indexName, String interval) { + String aggrQuery = query.get(Constants.JsonPaths.AGGREGATION_QUERY).asText(); + if(interval!=null && !interval.isEmpty()) + aggrQuery = aggrQuery.replace(Constants.JsonPaths.INTERVAL_VAL, interval); + String rqMs = query.get(Constants.JsonPaths.REQUEST_QUERY_MAP).asText(); + String dateReferenceField = query.get(Constants.JsonPaths.DATE_REF_FIELD).asText(); + JsonNode requestQueryMaps = null; + ObjectNode objectNode = null; + ObjectMapper mapper = new ObjectMapper(); + Map esFilterMap = new HashMap<>(); + try { + requestQueryMaps = new ObjectMapper().readTree(rqMs); + request.setEsFilters(esFilterMap); + if(query.get(Constants.JsonPaths.MODULE).asText().equals(Constants.Modules.COMMON) && + !request.getModuleLevel().equals(Constants.Modules.HOME_REVENUE) && + !request.getModuleLevel().equals(Constants.Modules.HOME_SERVICES)) { + request.getFilters().put(Constants.Filters.MODULE, request.getModuleLevel()); + } + Iterator> filtersItr = request.getFilters().entrySet().iterator(); + while(filtersItr.hasNext()) { + Entry entry = filtersItr.next(); + if(null != requestQueryMaps.get(entry.getKey()) && !String.valueOf(entry.getValue()).equals(Constants.Filters.FILTER_ALL)) { + String esQueryKey = requestQueryMaps.get(entry.getKey()).asText(); + request.getEsFilters().put(esQueryKey, entry.getValue()); + } + } + ElasticSearchDictator dictator = elasticSearchDao.createSearchDictatorV2(request, indexName, "", dateReferenceField); + SearchRequest searchRequest = elasticSearchDao.buildElasticSearchQuery(dictator); + JsonNode querySegment = mapper.readTree(searchRequest.source().toString()); + objectNode = (ObjectNode) querySegment; + JsonNode aggrNode = mapper.readTree(aggrQuery).get(Constants.JsonPaths.AGGS); + objectNode.put(Constants.JsonPaths.AGGS, mapper.readTree(aggrQuery).get(Constants.JsonPaths.AGGS)); + } catch (Exception ex) { + logger.error("Encountered an Exception while parsing the JSON : " + ex.getMessage()); + } + return objectNode; + + } + +} diff --git a/dashboard-analytics/src/main/java/com/tarento/analytics/service/impl/RestService.java b/dashboard-analytics/src/main/java/com/tarento/analytics/service/impl/RestService.java new file mode 100644 index 000000000..bf87841a6 --- /dev/null +++ b/dashboard-analytics/src/main/java/com/tarento/analytics/service/impl/RestService.java @@ -0,0 +1,158 @@ +package com.tarento.analytics.service.impl; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.tomcat.util.codec.binary.Base64; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.http.*; +import org.springframework.stereotype.Component; +import org.springframework.web.client.HttpClientErrorException; +import org.springframework.web.client.RestTemplate; + +import java.io.IOException; +import java.nio.charset.Charset; +import java.util.ArrayList; +import java.util.List; + +import static javax.servlet.http.HttpServletRequest.BASIC_AUTH; +import static org.apache.commons.codec.CharEncoding.US_ASCII; +import static org.springframework.http.HttpHeaders.AUTHORIZATION; + +@Component +public class RestService { + public static final Logger LOGGER = LoggerFactory.getLogger(RestService.class); + + @Value("${services.esindexer.host}") + private String indexServiceHost; + @Value("${egov.services.esindexer.host.search}") + private String indexServiceHostSearch; + @Value("${services.esindexer.host}") + private String dssindexServiceHost; + @Value("${egov.es.username}") + private String userName; + @Value("${egov.es.password}") + private String password; + + @Autowired + private RetryTemplate retryTemplate; + + + /** + * search on Elastic search for a search query + * @param index elastic search index name against which search operation + * @param searchQuery search query as request body + * @return + * @throws IOException + */ + public JsonNode search(String index, String searchQuery) { + + String url =( indexServiceHost) + index + indexServiceHostSearch; + HttpHeaders headers = getHttpHeaders(); + headers.setContentType(MediaType.APPLICATION_JSON); + LOGGER.info("Index Name : " + index); + LOGGER.info("Searching ES for Query: " + searchQuery); + HttpEntity requestEntity = new HttpEntity<>(searchQuery, headers); + String reqBody = requestEntity.getBody(); + JsonNode responseNode = null; + + try { + ResponseEntity response = retryTemplate.postForEntity(url, requestEntity); + responseNode = new ObjectMapper().convertValue(response.getBody(), JsonNode.class); + LOGGER.info("RestTemplate response :- "+responseNode); + + } catch (HttpClientErrorException e) { + e.printStackTrace(); + LOGGER.error("client error while searching ES : " + e.getMessage()); + } + return responseNode; + } + + /** + * makes a client rest api call of Http POST option + * @param uri + * @param authToken + * @param requestNode + * @return + * @throws IOException + */ + public JsonNode post(String uri, String authToken, JsonNode requestNode) { + + HttpHeaders headers = new HttpHeaders(); + if(authToken != null && !authToken.isEmpty()) + headers.add("Authorization", "Bearer "+ authToken ); + headers.setContentType(MediaType.APPLICATION_JSON); + + LOGGER.info("Request Node: " + requestNode); + HttpEntity requestEntity = null; + if(requestNode != null ) requestEntity = new HttpEntity<>(requestNode.toString(), headers); + else requestEntity = new HttpEntity<>("{}", headers); + + JsonNode responseNode = null; + + try { + ResponseEntity response = retryTemplate.postForEntity(uri,requestEntity); + responseNode = new ObjectMapper().convertValue(response.getBody(), JsonNode.class); + LOGGER.info("RestTemplate response :- "+responseNode); + + } catch (HttpClientErrorException e) { + LOGGER.error("post client exception: " + e.getMessage()); + } + return responseNode; + } + + /** + * makes a client rest api call of Http GET option + * @param uri + * @param authToken + * @return + * @throws IOException + */ + public JsonNode get(String uri, String authToken) { + + HttpHeaders headers = new HttpHeaders(); + headers.add("Authorization", "Bearer "+ authToken ); + headers.setContentType(MediaType.APPLICATION_JSON); + HttpEntity headerEntity = new HttpEntity<>("{}", headers); + + JsonNode responseNode = null; + try { + ResponseEntity response = retryTemplate.getForEntity(uri, headerEntity); + responseNode = new ObjectMapper().convertValue(response.getBody(), JsonNode.class); + LOGGER.info("RestTemplate response :- "+responseNode); + + } catch (HttpClientErrorException e) { + LOGGER.error("get client exception: " + e.getMessage()); + } + return responseNode; + } + + + private HttpHeaders getHttpHeaders() { + HttpHeaders headers = new HttpHeaders(); + headers.add(AUTHORIZATION, getBase64Value(userName, password)); + headers.setContentType(MediaType.APPLICATION_FORM_URLENCODED); + headers.setContentType(MediaType.APPLICATION_JSON); + + List mediaTypes = new ArrayList<>(); + mediaTypes.add(MediaType.APPLICATION_JSON); + headers.setAccept(mediaTypes); + return headers; + } + + /** + * Helper Method to create the Base64Value for headers + * + * @param userName + * @param password + * @return + */ + private String getBase64Value(String userName, String password) { + String authString = String.format("%s:%s", userName, password); + byte[] encodedAuthString = Base64.encodeBase64(authString.getBytes(Charset.forName(US_ASCII))); + return String.format(BASIC_AUTH, new String(encodedAuthString)); + } + +} diff --git a/dashboard-analytics/src/main/java/com/tarento/analytics/service/impl/RetryTemplate.java b/dashboard-analytics/src/main/java/com/tarento/analytics/service/impl/RetryTemplate.java new file mode 100644 index 000000000..e4f1f59f8 --- /dev/null +++ b/dashboard-analytics/src/main/java/com/tarento/analytics/service/impl/RetryTemplate.java @@ -0,0 +1,38 @@ +package com.tarento.analytics.service.impl; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.http.HttpEntity; +import org.springframework.http.HttpMethod; +import org.springframework.http.ResponseEntity; +import org.springframework.retry.annotation.Backoff; +import org.springframework.retry.annotation.Retryable; +import org.springframework.stereotype.Component; +import org.springframework.web.client.ResourceAccessException; +import org.springframework.web.client.RestTemplate; +import javax.naming.ServiceUnavailableException; + +/** + * Wraps rest template with retry + */ +@Component +public class RetryTemplate { + + @Autowired + private RestTemplate restTemplate; + + @Retryable(value = {RuntimeException.class, ResourceAccessException.class, ServiceUnavailableException.class}, + maxAttemptsExpression = "#{${service.retry.maxAttempts}}", + backoff = @Backoff(delayExpression = "#{${service.retry.backoff.delay}}")) + public ResponseEntity postForEntity(String url, Object request) { + return restTemplate.postForEntity(url, request, Object.class); + } + + @Retryable(value = {RuntimeException.class, ResourceAccessException.class, ServiceUnavailableException.class}, + maxAttemptsExpression = "#{${service.retry.maxAttempts}}", + backoff = @Backoff(delayExpression = "#{${service.retry.backoff.delay}}")) + public ResponseEntity getForEntity(String url, HttpEntity headerEntity) { + return restTemplate.exchange(url, HttpMethod.GET, headerEntity, Object.class); + } + +} + diff --git a/dashboard-analytics/src/main/java/com/tarento/analytics/utils/ElasticProperties.java b/dashboard-analytics/src/main/java/com/tarento/analytics/utils/ElasticProperties.java new file mode 100644 index 000000000..c27e7b79a --- /dev/null +++ b/dashboard-analytics/src/main/java/com/tarento/analytics/utils/ElasticProperties.java @@ -0,0 +1,73 @@ +package com.tarento.analytics.utils; + +public interface ElasticProperties { + + public interface Property { + final String TRANSACTION_TRANSDATE = "TransDate"; + final String TRANSACTION_STOREID = "StoreId"; + final String TRANSACTION_NETAMOUNT = "NetAmt"; + final String TARGET_TARGETDATE = "targetDate"; + final String TARGET_STORECODE = "storeCode"; + final String TARGET_VERSION = "version"; + final String TARGET_SALESVALUE = "salesValue"; + final String TARGET_AVGVALUE = "avgValue"; + final String TARGET_RECEIPTVALUE = "receiptsValue"; + final String ITEM_DETAILS_CATEGORY_ID = "itemDetails.categoryId"; + final String ITEM_DETAILS_PRICE = "itemDetails.price" ; + final String AVERAGE_RATING = "AvgRating"; + final String SORT_ORDER_DESCENDING = "desc"; + final String SORT_ORDER_ASCENDING = "asc"; + final String COUNTS = "counts"; + final String COUNTS_RATING = "CountsRating"; + final String FEEDBACK_VALUE_RESPONSE = "feedback_value"; + } + + public interface SuccessMessages { + final String STORE_USER = "User and Store successfully mapped" ; + final String STORE_TIMING = "Store Times entered successfully" ; + } + + public interface Query { + final String NESTED = "NESTED"; + final String MATCH_CONDITION = "MATCH"; + final String RANGE_CONDITION = "RANGE"; + final String AGGREGATION_CONDITION = "AGGREGATION"; + final String TRANSACTION_DATE_FIELD = "transDate"; + final String FEEDBACK_DATE_TIME = "serverDateTime"; + final String COUNT_STORE_CODE = "storeId"; + final String COUNT_RATING_VALUE = "value"; + final String COUNT_REASON_KEYWORD = "reasons.keyword"; + final String COUNT_GENDER_KEYWORD = "gender.keyword"; + final String COUNT_AGEGROUP_KEYWORD = "ageGroup.keyword"; + final String SUM = "SUM"; + final String VALUE_COUNT="value_count"; + final String AVG = "AVG"; + final String CUMMULATIVE_SUM="cumulative_sum"; + final String FIELD = "FIELD"; + final String COUNT = "COUNT"; + final String BUCKETS_PATH = "BUCKETS_PATH"; + final String DATE_HISTOGRAM = "DATE_HISTOGRAM"; + final String EXTENDED_BOUNDS = "EXTENDED_BOUNDS"; + final String PATH ="PATH"; + final String MIN = "MIN"; + final String MAX = "MAX"; + final String INTERVAL = "INTERVAL"; + final String HOUR ="HOUR"; + final String DAY ="DAY"; + final String DAY_OF_WEEK ="dayOfWeek"; + final String ASC="ASC"; + final String DESC ="DESC"; + + final String MINUTE ="MINUTE"; + final String MONTH ="MONTH"; + final String TERM ="TERMS"; + final String SIZE="SIZE"; + final String ORDER="ORDER"; + final Integer TOP_CSAT_STORE_COUNT = 5; + final String LABEL="label"; + + + + } + +} \ No newline at end of file diff --git a/dashboard-analytics/src/main/java/com/tarento/analytics/utils/ElasticSearchClient.java b/dashboard-analytics/src/main/java/com/tarento/analytics/utils/ElasticSearchClient.java new file mode 100644 index 000000000..b25f72285 --- /dev/null +++ b/dashboard-analytics/src/main/java/com/tarento/analytics/utils/ElasticSearchClient.java @@ -0,0 +1,48 @@ +package com.tarento.analytics.utils; + +import org.apache.http.HttpHost; +import org.elasticsearch.client.RestClient; +import org.elasticsearch.client.RestHighLevelClient; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; + + +@Component +public class ElasticSearchClient { + private final String hostName; + private final Integer port; + private final static String HTTP = "http"; + private final Integer topPerformanceCount; + private final String transactionIndex; + private final String transactionType; + private final String targetIndex; + private final String targetType; + + public ElasticSearchClient(@Value("${services.esindexer.host.name}") String hostName, + @Value("${services.esindexer.host.port}") Integer port, + @Value("${top.performance.record.count}") Integer topPerformanceCount, + @Value("${es.index.name}") String transactionIndex, + @Value("${es.document.type}") String transactionType, + @Value("${es.target.index.name}") String targetIndex, + @Value("${es.target.document.type}") String targetType){ + + this.hostName = hostName; + this.port = port; + this.topPerformanceCount = topPerformanceCount; + this.transactionIndex = transactionIndex; + this.transactionType = transactionType; + this.targetIndex = targetIndex; + this.targetType = targetType; + + } + + public RestHighLevelClient getClient() { + return new RestHighLevelClient( + RestClient.builder(new HttpHost(hostName, port,HTTP))); + + } + + + + +} diff --git a/dashboard-analytics/src/main/java/com/tarento/analytics/utils/JSONObjectUtil.java b/dashboard-analytics/src/main/java/com/tarento/analytics/utils/JSONObjectUtil.java new file mode 100644 index 000000000..6a7e33e2c --- /dev/null +++ b/dashboard-analytics/src/main/java/com/tarento/analytics/utils/JSONObjectUtil.java @@ -0,0 +1,41 @@ +package com.tarento.analytics.utils; +import org.springframework.beans.factory.annotation.Autowired; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.gson.Gson; + +public class JSONObjectUtil { + @Autowired + public ObjectMapper mapper; + @Autowired + public Gson gson; + + + /** + * @return + */ + public static String getJsonString(ObjectMapper objectMapper,Object object) throws JsonProcessingException { + if(objectMapper != null){ + return objectMapper.writeValueAsString(object); + } + return null; + } + + public ObjectMapper getMapper() { + return mapper; + } + + public void setObjectMapper(ObjectMapper objectMapper){ + mapper=objectMapper; + } + + public Gson getGson() { + return gson; + } + + public void setGson(Gson gsonn) + { + gson = gsonn; + } +} diff --git a/dashboard-analytics/src/main/java/com/tarento/analytics/utils/JsonKey.java b/dashboard-analytics/src/main/java/com/tarento/analytics/utils/JsonKey.java new file mode 100644 index 000000000..d60097bc7 --- /dev/null +++ b/dashboard-analytics/src/main/java/com/tarento/analytics/utils/JsonKey.java @@ -0,0 +1,20 @@ +/** + * + */ +package com.tarento.analytics.utils; + +/** + * @author Satish + * + */ +public class JsonKey { + + public static final String STATUS_CODE = "statusCode"; + public static final String STATUS = "statusInfo"; + public static final String STATUS_MESSAGE = "statusMessage"; + public static final String ERROR_MESSAGE = "errorMessage"; + + public static final String RESPONSE_DATA = "responseData"; + + +} diff --git a/dashboard-analytics/src/main/java/com/tarento/analytics/utils/PathRoutes.java b/dashboard-analytics/src/main/java/com/tarento/analytics/utils/PathRoutes.java new file mode 100644 index 000000000..75e90b40c --- /dev/null +++ b/dashboard-analytics/src/main/java/com/tarento/analytics/utils/PathRoutes.java @@ -0,0 +1,44 @@ +package com.tarento.analytics.utils; + +public interface PathRoutes { + + public interface DashboardApi { + final String DASHBOARD_ROOT_PATH = "/dashboard"; + + final String TEST_PATH = "/test"; + final String FILE_PATH = "/files"; + + final String GET_CHART = "/getChart"; + final String GET_CHART_V2 = "/getChartV2"; + final String GET_CHART_V3 = "/getChartV3"; + final String GET_DASHBOARD_CONFIG = "/getDashboardConfig"; + final String GET_HOME_CONFIG = "/getHomeConfig"; + final String GET_ALL_VISUALIZATIONS = "/getAllVisualizations"; + final String ADD_NEW_DASHBOARD = "/addNewDashboard"; + final String MAP_DASHBOARD_VISUALIZATIOn = "/mapVisualizationToDashboard"; + final String MAP_VISUALIZATION_ROLE = "/mapVisualizationToRole"; + final String GET_HEADER_DATA = "/getDashboardHeader"; + final String GET_FEEDBACK_MESSAGE="/getPulseFeedbackMessage"; + final String TARGET_DISTRICT_ULB = "/targetDistrict"; + + } + + public interface MetadataApi { + final String METADATA_ROOT_PATH = "/meta"; + + final String GET_CATEGORIES = "/getCategories"; + final String GET_SUB_CATEGORIES = "/getSubCategories"; + final String GET_ITEMS = "/getItems"; + final String GET_MASTERS = "/getMasters"; + final String FLUSH_MASTERS = "/flushMasters"; + final String GET_PULSE_RATING_CONFIG="/getPulseRatingConfig"; + final String GET_RATING_CONFIG = "/getRatingConfiguration"; + final String GET_RATING_CONFIG_ENCODE = "/getRatingConfig"; + final String GET_CONFIG_VERSION = "/getConfigVersion"; + final String PULSE_VERIFY_ORG_PIN = "/verifyOrgs"; + final String PUT_ORG_INDEX ="/putIndex"; + final String PUT_QUERY ="/createQuery"; + + + } +} diff --git a/dashboard-analytics/src/main/java/com/tarento/analytics/utils/ResponseCode.java b/dashboard-analytics/src/main/java/com/tarento/analytics/utils/ResponseCode.java new file mode 100644 index 000000000..ac4dc3962 --- /dev/null +++ b/dashboard-analytics/src/main/java/com/tarento/analytics/utils/ResponseCode.java @@ -0,0 +1,87 @@ +package com.tarento.analytics.utils; + +import com.tarento.analytics.constant.Constants; + +/** + * + * @author Abhishek + * + */ +public enum ResponseCode { + UnAuthorised(Constants.UNAUTHORIZED_ID, Constants.UNAUTHORIZED), Success( + Constants.SUCCESS_ID, Constants.SUCCESS),FAILURE( + Constants.FAILURE_ID, Constants.PROCESS_FAIL); + /** + * error code contains int value + */ + private int errorCode; + /** + * errorMessage contains proper error message. + */ + private String errorMessage; + + + + /** + * @param errorCode + * @param errorMessage + */ + private ResponseCode(int errorCode, String errorMessage) { + this.errorCode = errorCode; + this.errorMessage = errorMessage; + } + + /** + * + * @param errorCode + * @return + */ + public String getMessage(int errorCode) { + return ""; + } + + /** + * @return + */ + public int getErrorCode() { + return errorCode; + } + + /** + * @param errorCode + */ + public void setErrorCode(int errorCode) { + this.errorCode = errorCode; + } + + /** + * @return + */ + public String getErrorMessage() { + return errorMessage; + } + + /** + * @param errorMessage + */ + public void setErrorMessage(String errorMessage) { + this.errorMessage = errorMessage; + } + + /** + * This method will provide status message based on code + * + * @param code + * @return String + */ + public static String getResponseMessage(int code) { + String value = ""; + ResponseCode responseCodes[] = ResponseCode.values(); + for (ResponseCode actionState : responseCodes) { + if (actionState.getErrorCode() == code) { + value = actionState.getErrorMessage(); + } + } + return value; + } +} diff --git a/dashboard-analytics/src/main/java/com/tarento/analytics/utils/ResponseGenerator.java b/dashboard-analytics/src/main/java/com/tarento/analytics/utils/ResponseGenerator.java new file mode 100644 index 000000000..ed7a89845 --- /dev/null +++ b/dashboard-analytics/src/main/java/com/tarento/analytics/utils/ResponseGenerator.java @@ -0,0 +1,86 @@ +package com.tarento.analytics.utils; + + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; + +public class ResponseGenerator { + + + private static ObjectMapper objectMapper = new ObjectMapper(); + + public static String failureResponse() throws JsonProcessingException{ + ObjectNode response = objectMapper.createObjectNode(); + response.put(JsonKey.STATUS_CODE, ResponseCode.FAILURE.getErrorCode()); + response.put(JsonKey.STATUS_MESSAGE, + ResponseCode.FAILURE.getErrorMessage()); + response.put(JsonKey.ERROR_MESSAGE, + ResponseCode.FAILURE.getErrorMessage()); + return JSONObjectUtil.getJsonString(objectMapper,response); + } + + + public static String failureResponse(String message) throws JsonProcessingException{ + ObjectNode actualResponse = objectMapper.createObjectNode(); + + ObjectNode response = objectMapper.createObjectNode(); + response.put(JsonKey.STATUS_CODE, ResponseCode.FAILURE.getErrorCode()); + response.put(JsonKey.STATUS_MESSAGE, + ResponseCode.FAILURE.getErrorMessage()); + response.put(JsonKey.ERROR_MESSAGE,message); + actualResponse.putPOJO(JsonKey.STATUS,response); + + return JSONObjectUtil.getJsonString(objectMapper,actualResponse); + } + + public static String failureResponse(String errorCode, String message) throws JsonProcessingException{ + ObjectNode actualResponse = objectMapper.createObjectNode(); + + ObjectNode response = objectMapper.createObjectNode(); + response.put(errorCode, message); + response.put(JsonKey.STATUS_MESSAGE, + message); + response.put(JsonKey.ERROR_MESSAGE,message); + actualResponse.putPOJO(JsonKey.STATUS,response); + + return JSONObjectUtil.getJsonString(objectMapper,actualResponse); + } + + public static String successResponse(Object obj) throws JsonProcessingException { + ObjectNode actualResponse = objectMapper.createObjectNode(); + + ObjectNode response = objectMapper.createObjectNode(); + response.put(JsonKey.STATUS_CODE, ResponseCode.Success.getErrorCode()); + response.put(JsonKey.STATUS_MESSAGE, ResponseCode.Success.getErrorMessage()); + response.put(JsonKey.ERROR_MESSAGE, ""); + actualResponse.putPOJO(JsonKey.STATUS,response); + if (obj != null) { + actualResponse.putPOJO(JsonKey.RESPONSE_DATA, obj); + } + + return JSONObjectUtil.getJsonString(objectMapper,actualResponse); + } + + /** + * this method will crate success response and send to controller. + * + * @return ObjectNode object. + */ + public static String successResponse(String message) throws JsonProcessingException{ + ObjectNode actualResponse = objectMapper.createObjectNode(); + ObjectNode response = objectMapper.createObjectNode(); + response.put(JsonKey.STATUS_CODE, ResponseCode.Success.getErrorCode()); + response.put(JsonKey.STATUS_MESSAGE, + ResponseCode.Success.getErrorMessage()); + //response.put(JsonKey.SUCCESS_MESSAGE, message); + actualResponse.putPOJO(JsonKey.STATUS,response); + response.put(JsonKey.ERROR_MESSAGE, ""); + actualResponse.putPOJO(JsonKey.STATUS,response); + if (message != null) { + actualResponse.putPOJO(JsonKey.RESPONSE_DATA, message); + } + + return JSONObjectUtil.getJsonString(objectMapper,actualResponse); + } +} diff --git a/dashboard-analytics/src/main/java/com/tarento/analytics/utils/ResponseRecorder.java b/dashboard-analytics/src/main/java/com/tarento/analytics/utils/ResponseRecorder.java new file mode 100644 index 000000000..3fa241914 --- /dev/null +++ b/dashboard-analytics/src/main/java/com/tarento/analytics/utils/ResponseRecorder.java @@ -0,0 +1,47 @@ +package com.tarento.analytics.utils; + +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +import org.apache.commons.lang3.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Component; + +import com.tarento.analytics.constant.Constants; + +@Component("responseRecorder") +public class ResponseRecorder { + + private static Logger logger = LoggerFactory.getLogger(ResponseRecorder.class); + public static volatile ConcurrentHashMap> requestModuleResponseMap = new ConcurrentHashMap<>(); + + public Object get(String visualizationCode, String module) { + if(StringUtils.isNotBlank(module)) { + module = Constants.Modules.COMMON.toString(); + } + if(requestModuleResponseMap.get(visualizationCode) != null && requestModuleResponseMap.get(visualizationCode).get(module) != null) { + return requestModuleResponseMap.get(visualizationCode).get(module); + } + return null; + } + + + public Boolean put(String visualizationCode, String module, Object object) { + if(StringUtils.isNotBlank(module)) { + module = Constants.Modules.COMMON.toString(); + } + if(requestModuleResponseMap.containsKey(visualizationCode)) { + Map innerMap = requestModuleResponseMap.get(visualizationCode); + innerMap.put(module, object); + } else { + Map innerMap = new HashMap<>(); + innerMap.put(module, object); + requestModuleResponseMap.put(visualizationCode, innerMap); + } + return Boolean.TRUE; + } + + +} diff --git a/dashboard-analytics/src/main/resources/application.properties b/dashboard-analytics/src/main/resources/application.properties new file mode 100644 index 000000000..518342bec --- /dev/null +++ b/dashboard-analytics/src/main/resources/application.properties @@ -0,0 +1,41 @@ +server.port=8080 +server.context-path=/dashboard-analytics + +spring.datasource.url=jdbc:mysql://localhost:3306/database +spring.datasource.username=root +spring.datasource.password=tiger +spring.datasource.driver-class-name=com.mysql.jdbc.Driver +spring.autoconfigure.exclude=org.springframework.boot.autoconfigure.jdbc.DataSourceAutoConfiguration, org.springframework.boot.autoconfigure.orm.jpa.HibernateJpaAutoConfiguration + +#----------------------------- ELASTIC SEARCH CONFIGURATIONS ------------------------------# +services.esindexer.host=http://104.211.240.72:9200/ +services.esindexer.host.name=104.211.240.72 +services.esindexer.host.port=9200 +services.esindexer.username=username +services.esindexer.password=password +es.index.name=_transaction +es.document.type=transaction +es.target.index.name=store_target +es.target.document.type=storeTarget +#egov.mdms-service.target.url=https://egov-micro-dev.egovernments.org/egov-mdms-service/v1/_get?masterName=tenants&moduleName=tenant&tenantId=pb +config.schema.paths=classpath*:schema/*.json + +#----------------------------- MISCELLANEOUS CONFIGURATION ----------------------------------# +top.performance.record.count=5 + +egov.services.esindexer.host.search=/_search + +egov.es.username=egov-admin +egov.es.password=TUSYns9mEcRPy77n + +#----------------------------- AMAZON S3 CREDENTIALS -------------------------------------------------# +aws.access.key.id = +aws.access.key.secret = +aws.region = ap-south-1 +aws.s3.audio.bucket = dss-project-bucket + +#---------------------------- MDMS ---------------------------------------------------------------------# +egov.mdms-service.target.url=https://mseva-uat.lgpunjab.gov.in/egov-mdms-service/v1/_search +#https://egov-micro-dev.egovernments.org/egov-mdms-service/v1/_search +egov.mdms-service.request ={"RequestInfo":{"authToken":""},"MdmsCriteria":{"tenantId":"pb","moduleDetails":[{"moduleName":"tenant","masterDetails":[{"name":"tenants"}]}]}} + diff --git a/dashboard-analytics/src/main/resources/application.properties.j2 b/dashboard-analytics/src/main/resources/application.properties.j2 new file mode 100644 index 000000000..2b63c7415 --- /dev/null +++ b/dashboard-analytics/src/main/resources/application.properties.j2 @@ -0,0 +1,75 @@ +server.port=8089 + + +spring.datasource.url={{ datasource_url | b64decode }} +spring.datasource.username={{ datasource_user | b64decode }} +spring.datasource.password={{ datasource_pwd | b64decode }} +spring.datasource.driver-class-name=com.mysql.jdbc.Driver + +#----------------------------- ELASTIC SEARCH CONFIGURATIONS ------------------------------# +services.esindexer.host={{ es_host | b64decode }} +services.esindexer.host.name={{ es_host_name | b64decode }} +services.esindexer.host.port=9200 +services.esindexer.username={{ es_user | b64decode }} +services.esindexer.password={{ es_pwd | b64decode }} +es.index.name=_transaction +es.document.type=transaction +es.transaction.index.name=_transaction +es.transaction.document.type=transaction +es.target.index.name=store_target +es.target.document.type=storeTarget +es.index.name.storeTarget=store_target +es.index.name.feedback=_app_feedback +es.index.name.storeFeedback=rain_feedback +es.document.type.storeTarget=storeTarget +es.document.type.feedback=appFeedback +es.document.type.storeFeedback=rainFeedback + +#---------------------------------Microservice details------------------------------------# +services.user.host.name=localhost +services.user.host.port=8081 +services.user.host.url=/user + +#----------------------------- MISCELLANEOUS CONFIGURATION ----------------------------------# +top.performance.store.count=5 + +#------------------------------ KAFKA CONFIGURATIONS ------------------------------# + +# KAFKA SERVER CONFIGURATIONS +kafka.config.bootstrap_server_config=localhost:9092 + +spring.kafka.consumer.value-deserializer=com.tarento.analytics.consumer.HashMapDeserializer +spring.kafka.consumer.key-deserializer=org.apache.kafka.common.serialization.StringDeserializer +spring.kafka.consumer.group-id=rain-transaction-chain +spring.kafka.producer.key-serializer=org.apache.kafka.common.serialization.StringSerializer +spring.kafka.producer.value-serializer=org.springframework.kafka.support.serializer.JsonSerializer + +# KAFKA CONSUMER CONFIGURATIONS +kafka.consumer.config.auto_commit=true +kafka.consumer.config.auto_commit_interval=100 +kafka.consumer.config.session_timeout=15000 +kafka.consumer.config.group_id=rain-transaction-chain +kafka.consumer.config.auto_offset_reset=earliest + +# KAFKA PRODUCER CONFIGURATIONS +kafka.producer.config.retries_config=0 +kafka.producer.config.batch_size_config=100000 +kafka.producer.config.linger_ms_config=100 +kafka.producer.config.buffer_memory_config=66554432 + +#--------------------------- KAFKA TOPICS TO CONSUME ---------------------------# + +kafka.topics.notify.newmessage.name=NewContentMessage +kafka.topics.notify.newmessage.key=content + +kafka.topics.transaction.pushdata.name=RainTransactionData +kafka.topics.transaction.pushdata.key=transactiondata + +kafka.topics.target.pushdata.name=NewTargetData +kafka.topics.target.pushdata.key=targetdata + +kafka.topics.feedback.pushdata.name=NewFeedbackData +kafka.topics.feedback.pushdata.key=feedbackdata + +kafka.topics.rating.pushdata.name=RainRatingData +kafka.topics.rating.pushdata.key=ratingdata \ No newline at end of file diff --git a/dashboard-analytics/src/main/resources/schema/ChartApiConfig.json b/dashboard-analytics/src/main/resources/schema/ChartApiConfig.json new file mode 100644 index 000000000..bfed52f3c --- /dev/null +++ b/dashboard-analytics/src/main/resources/schema/ChartApiConfig.json @@ -0,0 +1,2908 @@ +{ + "_comment": "COMMON NON-COLLECTION CHARTS", + "totalApplication": { + "chartName": "DSS_TOTAL_APPLICATION", + "queries": [ + { + "module": "PT", + "indexName": "ptindex-v1", + "aggrQuery": "{\"aggs\":{\"AGGR\":{\"filter\":{\"bool\":{\"must_not\":[{\"term\":{\"Data.tenantId.keyword\":\"pb.testing\"}}]}},\"aggs\":{\"Total Application\":{\"value_count\":{\"field\":\"Data.propertyId.keyword\"}}}}}}", + "requestQueryMap": "{\r\n \"district\" : \"Data.tenantData.city.districtCode\", \r\n\"tenantId\" : \"Data.tenantId\" \r\n}", + "dateRefField": "Data.@timestamp" + }, + { + "module": "TL", + "indexName": "tlindex-v1", + "aggrQuery": "{\"aggs\":{\"AGGR\":{\"filter\":{\"bool\":{\"must_not\":[{\"term\":{\"Data.tradelicense.tenantId.keyword\":\"pb.testing\"}}]}},\"aggs\":{\"Total Application\":{\"value_count\":{\"field\":\"Data.tradelicense.applicationnumber.keyword\"}}}}}}", + "requestQueryMap": "{\r\n \"district\" : \"Data.tenantData.city.districtCode\", \r\n\"tenantId\" : \"Data.tradelicense.tenantId\" \r\n}", + "dateRefField": "Data.tradelicense.applicationdate" + }, + { + "module": "PGR", + "indexName": "pgrindex-v1", + "aggrQuery": "{\"aggs\":{\"AGGR\":{\"filter\":{\"bool\":{\"must_not\":[{\"term\":{\"Data.tenantId.keyword\":\"pb.testing\"}}]}},\"aggs\":{\"Total Application\":{\"value_count\":{\"field\":\"Data.tenantId.keyword\"}}}}}}", + "requestQueryMap": "{\r\n \"district\" : \"Data.tenantData.city.districtCode\", \r\n\"tenantId\" : \"Data.tenantId\" \r\n}", + "dateRefField": "Data.dateOfComplaint" + } + ], + "chartType": "metric", + "valueType": "number", + "action": "", + "documentType": "_doc", + "drillChart": "none", + "aggregationPaths": [ + "Total Application" + ], + "insight": { + "chartResponseMap" : "totalApplication", + "action" : "differenceOfNumbers", + "upwardIndicator" : "positive", + "downwardIndicator" : "negative", + "textMessage" : "$indicator$value% than last $insightInterval", + "colorCode" : "#228B22", + "insightInterval" : "month" + }, + "_comment": " totalApplication is the chartId" + }, + "closedApplication": { + "chartName": "DSS_CLOSED_APPLICATION", + "queries": [ + { + "module": "PT", + "dateRefField": "Data.@timestamp", + "requestQueryMap": "{\r\n \"district\" : \"Data.tenantData.city.districtCode\", \r\n\"tenantId\" : \"Data.tenantId\" \r\n}", + "indexName": "ptindex-v1", + "aggrQuery": "{\"aggs\":{\"AGGR\":{\"filter\":{\"bool\":{\"must_not\":[{\"term\":{\"Data.tenantId.keyword\":\"pb.testing\"}}]}},\"aggs\":{\"Closed Application\":{\"filters\":{\"filters\":{\"closed\":{\"match\":{\"Data.status.keyword\":\"closed\"}},\"resolved\":{\"match\":{\"Data.status.keyword\":\"resolved\"}}}}}}}}}" + }, + { + "module": "TL", + "dateRefField": "Data.@timestamp", + "requestQueryMap": "{\r\n \"district\" : \"Data.tenantData.city.districtCode\", \r\n\"tenantId\" : \"Data.tenantId\" \r\n}", + "indexName": "tlindex-v1", + "aggrQuery": "{\"aggs\":{\"AGGR\":{\"filter\":{\"bool\":{\"must_not\":[{\"term\":{\"Data.tradelicense.tenantId.keyword\":\"pb.testing\"}}]}},\"aggs\":{\"Closed Application\":{\"filters\":{\"filters\":{\"closed\":{\"match\":{\"Data.status.keyword\":\"closed\"}},\"resolved\":{\"match\":{\"Data.status.keyword\":\"resolved\"}}}}}}}}}" + }, + { + "module": "PGR", + "dateRefField": "Data.@timestamp", + "requestQueryMap": "{\r\n \"district\" : \"Data.tenantData.city.districtCode\", \r\n\"tenantId\" : \"Data.tenantId\" \r\n}", + "indexName": "pgrindex-v1", + "aggrQuery": "{\"aggs\":{\"AGGR\":{\"filter\":{\"bool\":{\"must_not\":[{\"term\":{\"Data.tenantId.keyword\":\"pb.testing\"}}]}},\"aggs\":{\"Closed Application\":{\"filters\":{\"filters\":{\"closed\":{\"match\":{\"Data.status.keyword\":\"closed\"}},\"resolved\":{\"match\":{\"Data.status.keyword\":\"resolved\"}}}}}}}}}" + } + ], + "chartType": "metric", + "valueType": "number", + "action": "", + "documentType": "_doc", + "drillChart": "none", + "aggregationPaths": [ + "Closed Application" + ], + "insight": { + "chartResponseMap" : "closedApplication", + "action" : "differenceOfNumbers", + "upwardIndicator" : "positive", + "downwardIndicator" : "negative", + "textMessage" : "$indicator$value% than last $insightInterval", + "colorCode" : "#228B22", + "insightInterval" : "month" + }, + "_comment": " totalApplication is the chartId" + }, + "citizenRegistered": { + "chartName": "DSS_CITIZEN_REGISTERED", + "queries": [ + { + "module": "PGR", + "indexName": "pgrindex-v1", + "aggrQuery": "{\"aggs\":{\"AGGR\":{\"filter\":{\"bool\":{\"must_not\":[{\"term\":{\"Data.tenantId.keyword\":\"pb.testing\"}}]}},\"aggs\":{\"Citizen Registered\":{\"terms\":{\"field\":\"Data.citizen.uuid.keyword\"},\"aggs\":{\"type_count\":{\"cardinality\":{\"field\":\"Data.citizen.uuid.keyword\"}}}}}}}}", + "requestQueryMap": "{\"district\" : \"Data.tenantData.city.districtCode\", \"tenantId\" : \"Data.tenantId\", \"status\" : \"Data.status\"}", + "dateRefField": "Data.dateOfComplaint" + } + ], + "chartType": "metric", + "valueType": "number", + "action": "", + "documentType": "_doc", + "drillChart": "none", + "aggregationPaths": [ + "Citizen Registered" + ], + "insight": { + "chartResponseMap" : "citizenRegistered", + "action" : "differenceOfNumbers", + "upwardIndicator" : "positive", + "downwardIndicator" : "negative", + "textMessage" : "$indicator$value% than last $insightInterval", + "colorCode" : "#228B22", + "insightInterval" : "month" + }, + "_comment": " totalApplication is the chartId" + }, + "totalApplicationDeptWise": { + "chartName": "DSS_TOTAL_APPLICATIONS:_DEPARTMENT_WISE", + "queries": [ + { + "module": "PT", + "dateRefField": "Data.@timestamp", + "requestQueryMap": "{\r\n \"district\" : \"Data.tenantData.city.districtCode\", \r\n\"tenantId\" : \"Data.tenantId\" \r\n}", + "indexName": "ptindex-v1", + "aggrQuery": "{\"aggs\":{\"AGGR\":{\"filter\":{\"bool\":{\"must_not\":[{\"term\":{\"Data.tenantId.keyword\":\"pb.testing\"}}]}},\"aggs\":{\"Property Tax\":{\"value_count\":{\"field\":\"Data.tenantId.keyword\"}}}}}}" + }, + { + "module": "TL", + "dateRefField": "Data.@timestamp", + "requestQueryMap": "{\r\n \"district\" : \"Data.tenantData.city.districtCode\", \r\n\"tenantId\" : \"Data.tenantId\" \r\n}", + "indexName": "tlindex-v1", + "aggrQuery": "{\"aggs\":{\"AGGR\":{\"filter\":{\"bool\":{\"must_not\":[{\"term\":{\"Data.tradelicense.tenantId.keyword\":\"pb.testing\"}}]}},\"aggs\":{\"Trade Licence\":{\"value_count\":{\"field\":\"Data.tradelicense.tenantid.keyword\"}}}}}}" + }, + { + "module": "PGR", + "dateRefField": "Data.@timestamp", + "requestQueryMap": "{\r\n \"district\" : \"Data.tenantData.city.districtCode\", \r\n\"tenantId\" : \"Data.tenantId\" \r\n}", + "indexName": "pgrindex-v1", + "aggrQuery": "{\"aggs\":{\"AGGR\":{\"filter\":{\"bool\":{\"must_not\":[{\"term\":{\"Data.tenantId.keyword\":\"pb.testing\"}}]}},\"aggs\":{\"PGR\":{\"value_count\":{\"field\":\"Data.tenantId.keyword\"}}}}}}" + } + ], + "chartType": "pie", + "valueType": "number", + "action": "", + "documentType": "_doc", + "drillChart": "none", + "aggregationPaths": [ + "Property Tax", + "Trade Licence", + "PGR" + ], + "insight": { + }, + "_comment": " totalApplication is the chartId" + }, + "totalApplication&ClosedApplication": { + "chartName": "DSS_TOTAL_APPLICATION_&_CLOSED_APPLICATION", + "queries": [ + { + "module": "PT", + "dateRefField": "Data.@timestamp", + "requestQueryMap": "{\r\n \"district\" : \"Data.tenantData.city.districtCode\", \r\n\"tenantId\" : \"Data.tenantId\" \r\n}", + "indexName": "ptindex-v1", + "aggrQuery": "{\"aggs\":{\"AGGR\":{\"filter\":{\"bool\":{\"must_not\":[{\"term\":{\"Data.tenantId.keyword\":\"pb.testing\"}}]}},\"aggs\":{\"Total Application\":{\"date_histogram\":{\"field\":\"Data.@timestamp\",\"interval\":\"intervalvalue\"},\"aggs\":{\"Count\":{\"value_count\":{\"field\":\"Data.propertyId.keyword\"}}}},\"Closed Application\":{\"date_histogram\":{\"field\":\"Data.@timestamp\",\"interval\":\"intervalvalue\"},\"aggs\":{\"Applications Closed\":{\"filters\":{\"filters\":{\"closed\":{\"match\":{\"Data.status.keyword\":\"closed\"}}}},\"aggs\":{\"Count\":{\"value_count\":{\"field\":\"Data.propertyId.keyword\"}}}}}}}}}}" + }, + { + "module": "TL", + "dateRefField": "Data.@timestamp", + "requestQueryMap": "{\r\n \"district\" : \"Data.tenantData.city.districtCode\", \r\n\"tenantId\" : \"Data.tenantId\" \r\n}", + "indexName": "tlindex-v1", + "aggrQuery": "{\"aggs\":{\"AGGR\":{\"filter\":{\"bool\":{\"must_not\":[{\"term\":{\"Data.tradelicense.tenantId.keyword\":\"pb.testing\"}}]}},\"aggs\":{\"Total Application\":{\"date_histogram\":{\"field\":\"Data.tradelicense.applicationDate\",\"interval\":\"intervalvalue\"},\"aggs\":{\"Count\":{\"value_count\":{\"field\":\"Data.tradelicense.applicationnumber.keyword\"}}}},\"Closed Application\":{\"date_histogram\":{\"field\":\"Data.tradelicense.@timestamp\",\"interval\":\"intervalvalue\"},\"aggs\":{\"Applications Closed\":{\"filters\":{\"filters\":{\"closed\":{\"match\":{\"Data.status.keyword\":\"closed\"}},\"resolved\":{\"match\":{\"Data.status.keyword\":\"resolved\"}}}},\"aggs\":{\"Count\":{\"value_count\":{\"field\":\"Data.tradelicense.applicationnumber.keyword\"}}}}}}}}}}" + + }, + { + "module": "PGR", + "dateRefField": "Data.@timestamp", + "requestQueryMap": "{\r\n \"district\" : \"Data.tenantData.city.districtCode\", \r\n\"tenantId\" : \"Data.tenantId\" \r\n}", + "indexName": "pgrindex-v1", + "aggrQuery": "{\"aggs\":{\"AGGR\":{\"filter\":{\"bool\":{\"must_not\":[{\"term\":{\"Data.tenantId.keyword\":\"pb.testing\"}}]}},\"aggs\":{\"Total Application\":{\"date_histogram\":{\"field\":\"Data.dateOfComplaint\",\"interval\":\"intervalvalue\"},\"aggs\":{\"Count\":{\"value_count\":{\"field\":\"Data.tenantId.keyword\"}}}},\"Closed Application\":{\"date_histogram\":{\"field\":\"Data.dateOfComplaint\",\"interval\":\"intervalvalue\"},\"aggs\":{\"Applications Closed\":{\"filters\":{\"filters\":{\"closed\":{\"match\":{\"Data.status.keyword\":\"closed\"}}}},\"aggs\":{\"Count\":{\"value_count\":{\"field\":\"Data.tenantId.keyword\"}}}}}}}}}}" + + } + ], + "chartType": "line", + "valueType": "number", + "action": "", + "drillChart": "none", + "documentType": "_doc", + "aggregationPaths": [ + "Total Application", + "Closed Application" + ], + "isCumulative": false, + "interval": "month", + "insight": { + }, + "_comment": " " + }, + "topPerformingUlbsCompletionRate": { + "chartName": "DSS_PT_TOP_3_PERFORMING_ULBS", + "queries": [ + { + "module": "PGR", + "requestQueryMap": "{\"module\" : \"businessService.keyword\",\"tenantId\" : \"Data.tenantId\" , \"district\" : \"Data.tenantData.city.districtCode\"}", + "dateRefField": "Data.@timestamp", + "indexName": "pgrindex-v1", + "aggrQuery": "{\"aggs\":{\"AGGR\":{\"filter\":{\"bool\":{\"must_not\":[{\"term\":{\"Data.tenantId.keyword\":\"pb.testing\"}}]}},\"aggs\":{\"Closed Application\":{\"filter\":{\"term\":{\"Data.status.keyword\":\"resolved\"}},\"aggs\":{\"count\":{\"terms\":{\"field\":\"Data.tenantId.keyword\",\"order\":{\"_key\":\"asc\"},\"size\":100},\"aggs\":{\"tenant_count\":{\"value_count\":{\"field\":\"Data.tenantId.keyword\"}}}}}},\"Total Application\":{\"terms\":{\"field\":\"Data.tenantId.keyword\",\"order\":{\"_key\":\"asc\"},\"size\":100},\"aggs\":{\"tenant_count\":{\"value_count\":{\"field\":\"Data.tenantId.keyword\"}}}}}}}}" + }, + { + "module": "PT", + "requestQueryMap": "{\"module\" : \"businessService.keyword\",\"tenantId\" : \"Data.tenantId\" , \"district\" : \"Data.tenantData.city.districtCode\"}", + "dateRefField": "Data.@timestamp", + "indexName": "ptindex-v1", + "aggrQuery": "{\"aggs\":{\"AGGR\":{\"filter\":{\"bool\":{\"must_not\":[{\"term\":{\"Data.tenantId.keyword\":\"pb.testing\"}}]}},\"aggs\":{\"Closed Application\":{\"filter\":{\"exists\":{\"field\":\"Data.propertyId.keyword\"}},\"aggs\":{\"count\":{\"terms\":{\"field\":\"Data.tenantId.keyword\"},\"aggs\":{\"tenant_count\":{\"value_count\":{\"field\":\"Data.tenantId.keyword\"}}}}}},\"Total Application\":{\"terms\":{\"field\":\"Data.tenantId.keyword\"},\"aggs\":{\"tenant_count\":{\"value_count\":{\"field\":\"Data.tenantId.keyword\"}}}}}}}}" + }, + { + "module": "TL", + "requestQueryMap": "{\"module\" : \"businessService.keyword\",\"tenantId\" : \"Data.tenantData.code\" , \"district\" : \"Data.tenantData.city.districtCode\"}", + "dateRefField": "Data.tradelicense.applicationdate", + "indexName": "tlindex-v1", + "aggrQuery": "{\"aggs\":{\"AGGR\":{\"filter\":{\"bool\":{\"must_not\":[{\"term\":{\"Data.tradelicense.tenantId.keyword\":\"pb.testing\"}}]}},\"aggs\":{\"Closed Application\":{\"filter\":{\"exists\":{\"field\":\"Data.tradelicense.issueddate\"}},\"aggs\":{\"count\":{\"terms\":{\"field\":\"Data.tenantData.code.keyword\"},\"aggs\":{\"tenant_count\":{\"value_count\":{\"field\":\"Data.tenantData.code.keyword\"}}}}}},\"Total Application\":{\"filter\":{\"exists\":{\"field\":\"Data.tradelicense.applicationdate\"}},\"aggs\":{\"count\":{\"terms\":{\"field\":\"Data.tenantData.code.keyword\"},\"aggs\":{\"tenant_count\":{\"value_count\":{\"field\":\"Data.tenantData.code.keyword\"}}}}}}}}}}" + } + ], + "chartType": "perform", + "valueType": "percentage", + "drillChart": "none", + "documentType": "_doc", + "action": "percentage", + "plotLabel": "DSS_COMPLETION_RATE", + "order": "desc", + "limit": 3, + "aggregationPaths": [ + "Closed Application", + "Total Application" + ], + "insight": { + }, + "_comment": " Top Performing Ulbs for Completion rate" + }, + "bottomPerformingUlbsCompletionRate": { + "chartName": "DSS_PT_BOTTOM_3_PERFORMING_ULBS", + "queries": [ + { + "module": "PGR", + "requestQueryMap": "{\"module\" : \"businessService.keyword\",\"tenantId\" : \"Data.tenantId\" , \"district\" : \"Data.tenantData.city.districtCode\"}", + "dateRefField": "Data.@timestamp", + "indexName": "pgrindex-v1", + "aggrQuery": "{\"aggs\":{\"AGGR\":{\"filter\":{\"bool\":{\"must_not\":[{\"term\":{\"Data.tenantId.keyword\":\"pb.testing\"}}]}},\"aggs\":{\"Closed Application\":{\"filter\":{\"term\":{\"Data.status.keyword\":\"resolved\"}},\"aggs\":{\"count\":{\"terms\":{\"field\":\"Data.tenantId.keyword\"},\"aggs\":{\"tenant_count\":{\"value_count\":{\"field\":\"Data.tenantId.keyword\"}}}}}},\"Total Application\":{\"terms\":{\"field\":\"Data.tenantId.keyword\"},\"aggs\":{\"tenant_count\":{\"value_count\":{\"field\":\"Data.tenantId.keyword\"}}}}}}}}" + }, + { + "module": "PT", + "requestQueryMap": "{\"module\" : \"businessService.keyword\",\"tenantId\" : \"Data.tenantId\" ,\"district\" : \"Data.tenantData.city.districtCode\"}", + "dateRefField": "Data.@timestamp", + "indexName": "ptindex-v1", + "aggrQuery": "{\"aggs\":{\"AGGR\":{\"filter\":{\"bool\":{\"must_not\":[{\"term\":{\"Data.tenantId.keyword\":\"pb.testing\"}}]}},\"aggs\":{\"Closed Application\":{\"filter\":{\"exists\":{\"field\":\"Data.propertyId.keyword\"}},\"aggs\":{\"count\":{\"terms\":{\"field\":\"Data.tenantId.keyword\"},\"aggs\":{\"tenant_count\":{\"value_count\":{\"field\":\"Data.tenantId.keyword\"}}}}}},\"Total Application\":{\"terms\":{\"field\":\"Data.tenantId.keyword\"},\"aggs\":{\"tenant_count\":{\"value_count\":{\"field\":\"Data.tenantId.keyword\"}}}}}}}}" + }, + { + "module": "TL", + "requestQueryMap": "{\"module\" : \"businessService.keyword\",\"tenantId\" : \"Data.tenantData.code\" , \"district\" : \"Data.tenantData.city.districtCode\"}", + "dateRefField": "Data.tradelicense.applicationdate", + "indexName": "tlindex-v1", + "aggrQuery": "{\"aggs\":{\"AGGR\":{\"filter\":{\"bool\":{\"must_not\":[{\"term\":{\"Data.tradelicense.tenantId.keyword\":\"pb.testing\"}}]}},\"aggs\":{\"Closed Application\":{\"filter\":{\"exists\":{\"field\":\"Data.tradelicense.issueddate\"}},\"aggs\":{\"count\":{\"terms\":{\"field\":\"Data.tenantData.code.keyword\"},\"aggs\":{\"tenant_count\":{\"value_count\":{\"field\":\"Data.tenantData.code.keyword\"}}}}}},\"Total Application\":{\"filter\":{\"exists\":{\"field\":\"Data.tradelicense.applicationdate\"}},\"aggs\":{\"count\":{\"terms\":{\"field\":\"Data.tenantData.code.keyword\"},\"aggs\":{\"tenant_count\":{\"value_count\":{\"field\":\"Data.tenantData.code.keyword\"}}}}}}}}}}" + } + ], + "chartType": "perform", + "valueType": "percentage", + "drillChart": "none", + "documentType": "_doc", + "action": "percentage", + "plotLabel": "DSS_COMPLETION_RATE", + "order": "asc", + "limit": 3, + "aggregationPaths": [ + "Closed Application", + "Total Application" + ], + "insight": { + }, + "_comment": " Bottom Performing Ulbs for Completion rate" + }, + + "_comment": "COMMON COLLECTION CHARTS", + "cumulativeCollection": { + "chartName": "DSS_TOTAL_CUMULATIVE_COLLECTION", + "queries": [ + { + "module": "COMMON", + "dateRefField": "Data.Bill.billDetails.receiptDate", + "requestQueryMap": "{\r\n \"module\" : \"Data.Bill.billDetails.businessService.keyword\", \n\"tenantId\" : \"Data.tenantId\"}", + "indexName": "collectionsindex-v1", + "aggrQuery": "{\"aggs\":{\"AGGR\":{\"filter\":{\"bool\":{\"must_not\":[{\"term\":{\"Data.tenantId.keyword\":\"pb.testing\"}}],\"must\":[{\"terms\":{\"Data.Bill.billDetails.businessService.keyword\":[\"PT\",\"TL\"]}},{\"term\":{\"Data.Bill.billDetails.status.keyword\":\"Approved\"}}]}},\"aggs\":{\"Collections\":{\"date_histogram\":{\"field\":\"Data.Bill.billDetails.receiptDate\",\"interval\":\"intervalvalue\"},\"aggs\":{\"Sum\":{\"sum\":{\"field\":\"Data.Bill.billDetails.amountPaid\"}}}}}}}}" + } + ], + "chartType": "line", + "valueType": "amount", + "action": "", + "drillChart": "none", + "documentType": "_doc", + "aggregationPaths": [ + "Collections" + ], + "isCumulative": true, + "interval": "month", + "insight": { + }, + "_comment": " " + }, + "targetCollection": { + "chartName": "DSS_TARGET_COLLECTION", + "queries": [ + { + "module": "COMMON", + "requestQueryMap": "{\"module\" : \"businessService.keyword\", \"tenantId\" : \"tenantIdForMunicipalCorporation.keyword\"}", + "dateRefField": "", + "indexName": "dss-target_v1", + "aggrQuery": "{\"aggs\":{\"Target Collection\":{\"sum\":{\"field\":\"budgetProposedForMunicipalCorporation\"}}}}" + } + ], + "chartType": "metric", + "valueType": "amount", + "action": "", + "drillChart": "none", + "documentType": "_doc", + "aggregationPaths": [ + "Target Collection" + ], + "isDayUnit": true, + "postAggregationTheory" : "repsonseToDifferenceOfDates", + "insight": { + }, + "_comment": " " + }, + "targetAchieved": { + "chartName": "DSS_TARGET_ACHIEVED", + "queries": [ + { + "module": "COMMON", + "requestQueryMap": "{\r\n \"module\" : \"businessService.keyword\", \n\"tenantId\" : \"tenantIdForMunicipalCorporation\"}", + "dateRefField": "", + "indexName": "dss-target_v1", + "aggrQuery": "{\"aggs\":{\"Actual collection\":{\"sum\":{\"field\":\"budgetProposedForMunicipalCorporation\"}}}}" + }, + { + "module": "COMMON", + "requestQueryMap": "{\r\n \"district\" : \"dataObject.tenantData.cityDistrictCode\", \"module\" : \"Data.Bill.billDetails.businessService.keyword\", \n\"tenantId\" : \"Data.tenantId\"}", + "dateRefField": "Data.Bill.billDetails.receiptDate", + "indexName": "collectionsindex-v1", + "aggrQuery": "{\"aggs\":{\"AGGR\":{\"filter\":{\"bool\":{\"must_not\":[{\"term\":{\"Data.tenantId.keyword\":\"pb.testing\"}}],\"must\":[{\"term\":{\"Data.Bill.billDetails.status.keyword\":\"Approved\"}}]}},\"aggs\":{\"Total Collection\":{\"sum\":{\"field\":\"Data.Bill.billDetails.amountPaid\"}}}}}}" + } + + ], + "chartType": "metric", + "valueType": "percentage", + "drillChart": "none", + "documentType": "_doc", + "action": "percentage", + "aggregationPaths": [ + "Total Collection", + "Actual collection" + ], + "insight": { + "chartResponseMap" : "targetAchieved", + "action" : "differenceOfNumbers", + "upwardIndicator" : "positive", + "downwardIndicator" : "negative", + "textMessage" : "$indicator$value% than last $insightInterval", + "colorCode" : "#228B22", + "insightInterval" : "month" + }, + "_comment": " " + }, + "totalCollection": { + "chartName": "DSS_TOTAL_COLLECTION", + "queries": [ + { + "module": "PT", + "dateRefField": "Data.Bill.billDetails.receiptDate", + "requestQueryMap": "{\"module\" : \"Data.Bill.billDetails.businessService.keyword\", \"tenantId\" : \"Data.tenantId\", \"district\" : \"dataObject.tenantData.cityDistrictCode\"}", + "indexName": "collectionsindex-v1", + "aggrQuery": "{\"aggs\":{\"AGGR\":{\"filter\":{\"bool\":{\"must_not\":[{\"term\":{\"Data.tenantId.keyword\":\"pb.testing\"}}],\"must\":[{\"term\":{\"Data.Bill.billDetails.businessService.keyword\":\"PT\"}},{\"term\":{\"Data.Bill.billDetails.status.keyword\":\"Approved\"}}]}},\"aggs\":{\"Total Collection\":{\"sum\":{\"field\":\"Data.Bill.billDetails.amountPaid\"}}}}}}" + }, + { + "module": "TL", + "dateRefField": "Data.Bill.billDetails.receiptDate", + "requestQueryMap": "{\"module\" : \"Data.Bill.billDetails.businessService.keyword\", \"tenantId\" : \"Data.tenantId\", \"district\" : \"dataObject.tenantData.cityDistrictCode\"}", + "indexName": "collectionsindex-v1", + "aggrQuery": "{\"aggs\":{\"AGGR\":{\"filter\":{\"bool\":{\"must_not\":[{\"term\":{\"Data.tenantId.keyword\":\"pb.testing\"}}],\"must\":[{\"term\":{\"Data.Bill.billDetails.businessService.keyword\":\"TL\"}},{\"term\":{\"Data.Bill.billDetails.status.keyword\":\"Approved\"}}]}},\"aggs\":{\"Total Collection\":{\"sum\":{\"field\":\"Data.Bill.billDetails.amountPaid\"}}}}}}" + } + ], + "chartType": "metric", + "valueType": "Amount", + "drillChart": "none", + "documentType": "_doc", + "action": "", + "aggregationPaths": [ + "Total Collection" + ], + "insight": { + "chartResponseMap" : "totalCollection", + "action" : "differenceOfNumbers", + "upwardIndicator" : "positive", + "downwardIndicator" : "negative", + "textMessage" : "$indicator$value% than last $insightInterval", + "colorCode" : "#228B22", + "insightInterval" : "month" + }, + "_comment": " " + }, + "todaysCollection": { + "chartName": "DSS_TOTAL_COLLECTION_TODAY", + "queries": [ + { + "module": "PT", + "dateRefField": "", + "requestQueryMap": "{\"module\" : \"Data.Bill.billDetails.businessService.keyword\", \"tenantId\" : \"Data.tenantId\", \"district\" : \"dataObject.tenantData.cityDistrictCode\"}", + "indexName": "collectionsindex-v1", + "aggrQuery": "{\"aggs\":{\"AGGR\":{\"filter\":{\"bool\":{\"must_not\":[{\"term\":{\"Data.tenantId.keyword\":\"pb.testing\"}}],\"must\":[{\"term\":{\"Data.Bill.billDetails.status.keyword\":\"Approved\"}},{\"term\":{\"Data.Bill.billDetails.businessService.keyword\":\"PT\"}},{\"range\":{\"Date.@timestamp\":{\"gte\":\"now-24h\",\"lte\":\"now\"}}}]}},\"aggs\":{\"Total collection\":{\"sum\":{\"field\":\"dataObject.Bill.billDetails.amountPaid\"}}}}}}" + }, + { + "module": "TL", + "dateRefField": "", + "requestQueryMap": "{\"module\" : \"Data.Bill.billDetails.businessService.keyword\", \"tenantId\" : \"Data.tenantId\", \"district\" : \"dataObject.tenantData.cityDistrictCode\"}", + "indexName": "collectionsindex-v1", + "aggrQuery": "{\"aggs\":{\"AGGR\":{\"filter\":{\"bool\":{\"must_not\":[{\"term\":{\"Data.tenantId.keyword\":\"pb.testing\"}}],\"must\":[{\"term\":{\"Data.Bill.billDetails.status.keyword\":\"Approved\"}},{\"term\":{\"Data.Bill.billDetails.businessService.keyword\":\"TL\"}},{\"range\":{\"Date.@timestamp\":{\"gte\":\"now-24h\",\"lte\":\"now\"}}}]}},\"aggs\":{\"Total collection\":{\"sum\":{\"field\":\"dataObject.Bill.billDetails.amountPaid\"}}}}}}" + } + ], + "chartType": "metric", + "valueType": "Amount", + "drillChart": "none", + "documentType": "_doc", + "action": "", + "aggregationPaths": [ + "Total Collection" + ], + "insight": { + "chartResponseMap" : "todaysCollection", + "action" : "differenceOfNumbers", + "upwardIndicator" : "positive", + "downwardIndicator" : "negative", + "textMessage" : "$indicator$value% than last $insightInterval", + "colorCode" : "#228B22", + "insightInterval" : "month" + }, + "_comment": " " + }, + "totalCollectionDeptWise": { + "chartName": "DSS_TOTAL_CUMULATIVE_COLLECTION:_DEPARTMENT_WISE", + "queries": [ + { + "module": "COMMON", + "requestQueryMap": "{\"module\" : \"Data.Bill.billDetails.businessService.keyword\", \"tenantId\" : \"Data.tenantId\", \"district\" : \"dataObject.tenantData.cityDistrictCode\"}", + "dateRefField": "Data.Bill.billDetails.receiptDate", + "indexName": "collectionsindex-v1", + "aggrQuery": "{\"aggs\":{\"AGGR\":{\"filter\":{\"bool\":{\"must_not\":[{\"term\":{\"Data.tenantId.keyword\":\"pb.testing\"}}],\"must\":[{\"term\":{\"Data.Bill.billDetails.status.keyword\":\"Approved\"}},{\"terms\":{\"Data.Bill.billDetails.businessService.keyword\":[\"PT\",\"TL\"]}}]}},\"aggs\":{\"Business Service\":{\"terms\":{\"field\":\"Data.Bill.billDetails.businessService.keyword\"},\"aggs\":{\"total\":{\"sum\":{\"field\":\"Data.Bill.billDetails.amountPaid\"}}}}}}}}" + } + ], + "chartType": "pie", + "valueType": "Amount", + "drillChart": "none", + "documentType": "_doc", + "action": "", + "aggregationPaths": [ + "Business Service" + ], + "insight": { + }, + "_comment": " " + }, + "topPerformingUlbs": { + "chartName": "DSS_PT_TOP_3_PERFORMING_ULBS", + "queries": [ + { + "module": "COMMON", + "requestQueryMap": "{\r\n \"module\" : \"businessService.keyword\", \n\"tenantId\" : \"tenantIdForMunicipalCorporation\"}", + "dateRefField": "", + "indexName": "dss-target_v1", + "aggrQuery": "{\"aggs\":{\"Target Collection\":{\"terms\":{\"field\":\"tenantIdForMunicipalCorporation.keyword\",\"size\":\"200\",\"order\":{\"Sum\":\"desc\"}},\"aggs\":{\"Sum\":{\"sum\":{\"field\":\"budgetProposedForMunicipalCorporation\"}}}}}}" + }, + { + "module": "COMMON", + "requestQueryMap": "{\r\n \"module\" : \"Data.Bill.billDetails.businessService.keyword\", \n\"tenantId\" : \"Data.tenantId\"}", + "dateRefField": "Data.Bill.billDetails.receiptDate", + "indexName": "collectionsindex-v1", + "aggrQuery": "{\"aggs\":{\"AGGR\":{\"filter\":{\"bool\":{\"must_not\":[{\"term\":{\"Data.tenantId.keyword\":\"pb.testing\"}}],\"must\":[{\"term\":{\"Data.Bill.billDetails.status.keyword\":\"Approved\"}}]}},\"aggs\":{\"Total Collection\":{\"terms\":{\"field\":\"Data.tenantId.keyword\",\"size\":\"200\",\"order\":{\"Sum\":\"desc\"}},\"aggs\":{\"Sum\":{\"sum\":{\"field\":\"Data.Bill.billDetails.amountPaid\"}}}}}}}}" + } + ], + "chartType": "perform", + "valueType": "percentage", + "drillChart": "none", + "documentType": "_doc", + "action": "percentage", + "plotLabel": "DSS_TARGET_ACHIEVED", + "order": "desc", + "limit": 3, + "aggregationPaths": [ + "Total Collection","Target Collection" + ], + "insight": { + }, + "_comment": " Top Performing Ulbs for target achieved" + }, + "bottomPerformingUlbs": { + "chartName": "DSS_PT_BOTTOM_3_PERFORMING_ULBS", + "queries": [ + { + "module": "COMMON", + "requestQueryMap": "{\"module\" : \"businessService.keyword\",\"tenantId\" : \"tenantIdForMunicipalCorporation\" }", + "dateRefField": "", + "indexName": "dss-target_v1", + "aggrQuery": "{\"aggs\":{\"Target Collection\":{\"terms\":{\"field\":\"tenantIdForMunicipalCorporation.keyword\",\"size\":\"200\",\"order\":{\"Sum\":\"asc\"}},\"aggs\":{\"Sum\":{\"sum\":{\"field\":\"budgetProposedForMunicipalCorporation\"}}}}}}" + }, + { + "module": "COMMON", + "requestQueryMap": "{\"module\" : \"Data.Bill.billDetails.businessService.keyword\", \"tenantId\" : \"Data.tenantId\" }", + "dateRefField": "Data.Bill.billDetails.receiptDate", + "indexName": "collectionsindex-v1", + "aggrQuery": "{\"aggs\":{\"AGGR\":{\"filter\":{\"bool\":{\"must_not\":[{\"term\":{\"Data.tenantId.keyword\":\"pb.testing\"}}],\"must\":[{\"term\":{\"Data.Bill.billDetails.status.keyword\":\"Approved\"}}]}},\"aggs\":{\"Total Collection\":{\"terms\":{\"field\":\"Data.tenantId.keyword\",\"size\":\"200\",\"order\":{\"Sum\":\"asc\"}},\"aggs\":{\"Sum\":{\"sum\":{\"field\":\"Data.Bill.billDetails.amountPaid\"}}}}}}}}" + } + ], + "chartType": "perform", + "valueType": "percentage", + "drillChart": "none", + "documentType": "_doc", + "action": "percentage", + "plotLabel": "DSS_TARGET_ACHIEVED", + "order": "asc", + "limit": 3, + "aggregationPaths": [ + "Total Collection", "Target Collection" + ], + "insight": { + }, + "_comment": " Bottom Performing Ulbs for target achieved" + }, + "collectionByUsageType": { + "chartName": "DSS_PT_COLLECTION_BY_USAGE_TYPE", + "queries": [ + { + "module": "COMMON", + "requestQueryMap": "{\"module\" : \"dataObject.Bill.billDetails.businessService.keyword\", \"tenantId\" : \"dataObject.tenantId\", \"district\" : \"dataObject.tenantData.cityDistrictCode\" }", + "dateRefField": "dataObject.Bill.billDetails.receiptDate", + "indexName": "dss-collection_v1", + "aggrQuery": "{\"aggs\":{\"AGGR\":{\"filter\":{\"bool\":{\"must_not\":[{\"term\":{\"dataObject.tenantId.keyword\":\"pb.testing\"}}],\"must\":[{\"term\":{\"dataObject.Bill.billDetails.status.keyword\":\"Approved\"}}]}},\"aggs\":{\"Usage Type\":{\"terms\":{\"field\":\"domainObject.propertyDetails.units.usageCategoryMajor.keyword\"},\"aggs\":{\"Total Collection\":{\"sum\":{\"field\":\"dataObject.Bill.billDetails.amountPaid\"}}}}}}}}" + } + ], + "chartType": "pie", + "valueType": "amount", + "action": "", + "documentType": "_doc", + "drillChart": "none", + "aggregationPaths": [ + "Usage Type" + ], + "insight": { + }, + "_comment": " collection/amount per usage type" + }, + + "_comment": "PT NON-COLLECTION CHARTS", + "propertiesByUsageType": { + "chartName": "DSS_PT_PROPERTIES_BY_USAGE_TYPE", + "queries": [ + { + "module": "PT", + "requestQueryMap": "{\"module\" : \"Data.Bill.billDetails.businessService.keyword\", \"tenantId\" : \"Data.tenantId\", \"district\" : \"dataObject.tenantData.cityDistrictCode\"}", + "dateRefField": "Data.@timestamp", + "indexName": "ptindex-v1", + "aggrQuery": "{\"aggs\":{\"AGGR\":{\"filter\":{\"bool\":{\"must_not\":[{\"term\":{\"Data.tenantId.keyword\":\"pb.testing\"}}]}},\"aggs\":{\"Usage Type\":{\"terms\":{\"field\":\"Data.propertyDetails.units.usageCategoryMajor.keyword\"},\"aggs\":{\"Assessed Properties\":{\"value_count\":{\"field\":\"Data.propertyDetails.assessmentNumber.keyword\"}}}}}}}}" + + } + ], + "chartType": "pie", + "valueType": "number", + "action": "", + "documentType": "_doc", + "drillChart": "none", + "aggregationPaths": [ + "Usage Type" + ], + "insight": { + }, + "_comment": " properties having assessmentNumber per usage type" + }, + "propertiesAssessed": { + "chartName": "DSS_PT_TOTAL_PROPERTIES_ASSESSED", + "queries": [ + { + "module": "PT", + "indexName": "ptindex-v1", + "aggrQuery": "{\"aggs\":{\"AGGR\":{\"filter\":{\"bool\":{\"must_not\":[{\"term\":{\"Data.tenantId.keyword\":\"pb.testing\"}}]}},\"aggs\":{\"Assessed Properties\":{\"cardinality\":{\"field\":\"Data.propertyId.keyword\"}}}}}}", + "requestQueryMap": "{\"tenantId\" : \"Data.tenantId\" ,\"district\" : \"Data.tenantData.city.districtCode\"}", + "dateRefField": "Data.@timestamp" + } + ], + "chartType": "metric", + "valueType": "number", + "action": "", + "documentType": "_doc", + "drillChart": "none", + "aggregationPaths": [ + "Assessed Properties" + ], + "insight": { + "chartResponseMap" : "propertiesAssessed", + "action" : "differenceOfNumbers", + "upwardIndicator" : "positive", + "downwardIndicator" : "negative", + "textMessage" : "$indicator$value% than last $insightInterval", + "colorCode" : "#228B22", + "insightInterval" : "month" + }, + "_comment": " totol properties having assessmentNumber " + }, + "totalAssessment": { + "chartName": "DSS_PT_TOTAL_ASSESSMENTS", + "queries": [ + { + "module": "PT", + "indexName": "ptindex-v1", + "aggrQuery": "{\"aggs\":{\"AGGR\":{\"filter\":{\"bool\":{\"must_not\":[{\"term\":{\"Data.tenantId.keyword\":\"pb.testing\"}}]}},\"aggs\":{\"Assessed Properties\":{\"cardinality\":{\"field\":\"Data.propertyDetails.assessmentNumber.keyword\"}}}}}}", + + "requestQueryMap": "{\"tenantId\" : \"Data.tenantId\" ,\"district\" : \"Data.tenantData.city.districtCode\"}", + "dateRefField": "Data.@timestamp" + } + ], + "chartType": "metric", + "valueType": "number", + "action": "", + "documentType": "_doc", + "drillChart": "none", + "aggregationPaths": [ + "Assessed Properties" + ], + "insight": { + "chartResponseMap" : "totalAssessment", + "action" : "differenceOfNumbers", + "upwardIndicator" : "positive", + "downwardIndicator" : "negative", + "textMessage" : "$indicator$value% than last $insightInterval", + "colorCode" : "#228B22", + "insightInterval" : "month" + }, + "_comment": " totol properties having assessmentNumber " + }, + "cumulativePropertiesAssessed": { + "chartName": "DSS_PT_CUMULATIVE_PROPERTIES_ASSESSED", + "queries": [ + { + "module": "PT", + "indexName": "ptindex-v1", + "aggrQuery": "{\"aggs\":{\"AGGR\":{\"filter\":{\"bool\":{\"must_not\":[{\"term\":{\"Data.tenantId.keyword\":\"pb.testing\"}}]}},\"aggs\":{\"Collections\":{\"date_histogram\":{\"field\":\"Data.@timestamp\",\"interval\":\"intervalvalue\"},\"aggs\":{\"Count\":{\"cardinality\":{\"field\":\"Data.propertyDetails.assessmentNumber.keyword\"}}}}}}}}", + "requestQueryMap": "{\"tenantId\" : \"Data.tenantId\" ,\"district\" : \"Data.tenantData.city.districtCode\"}", + "dateRefField": "Data.@timestamp" + } + ], + "chartType": "line", + "valueType": "number", + "action": "", + "documentType": "_doc", + "drillChart": "none", + "aggregationPaths": [ + "Collections" + ], + "isCumulative": true, + "interval": "week", + "insight": { + }, + "_comment": " totol properties having assessmentNumber " + }, + "activeUlbs": { + "chartName": "DSS_PT_TOTAL_ACTIVE_ULBS", + "queries": [ + { + "module": "PT", + "requestQueryMap": "{\"tenantId\" : \"Data.tenantId\"}", + "dateRefField": "Data.@timestamp", + "indexName": "ptindex-v1", + "aggrQuery": "{\"aggs\":{\"AGGR\":{\"filter\":{\"bool\":{\"must_not\":[{\"term\":{\"Data.tenantId.keyword\":\"pb.testing\"}}]}},\"aggs\":{\"Active ULBs\":{\"terms\":{\"field\":\"Data.tenantId.keyword\"},\"aggs\":{\"type_count\":{\"cardinality\":{\"field\":\"Data.tenantId.keyword\"}}}}}}}}" + }, + { + "module": "TL", + "requestQueryMap": "{\"tenantId\" : \"Data.tenantData.code\"}", + "dateRefField": "Data.tradelicense.applicationdate", + "indexName": "tlindex-v1", + "aggrQuery": "{\"aggs\":{\"AGGR\":{\"filter\":{\"bool\":{\"must_not\":[{\"term\":{\"dataObject.tenantId.keyword\":\"pb.testing\"}}]}},\"aggs\":{\"Active ULBs\":{\"terms\":{\"field\":\"Data.tenantData.code.keyword\"},\"aggs\":{\"type_count\":{\"cardinality\":{\"field\":\"Data.tenantData.code.keyword\"}}}}}}}}" + } + ], + "chartType": "metric", + "valueType": "number", + "action": "", + "documentType": "_doc", + "drillChart": "none", + "aggregationPaths": [ + "Active ULBs" + ], + "insight": { + "chartResponseMap" : "activeUlbs", + "action" : "differenceOfNumbers", + "upwardIndicator" : "positive", + "downwardIndicator" : "negative", + "textMessage" : "$indicator$value% than last $insightInterval", + "colorCode" : "#228B22", + "insightInterval" : "month" + }, + "_comment": " total ULBs count" + }, + + "_comment": "PT COLLECTION CHARTS (Tax heads)", + "taxHeadsBreakupDDRRevenue": { + "chartName": "DSS_PT_TAX_HEAD_BREAKUP_REVENUE", + "queries": [ + { + "module": "COMMON", + "requestQueryMap": "{\"module\" : \"dataObject.Bill.billDetails.businessService.keyword\", \"usageId\" : \"domainObject.propertyDetails.usageCategoryMajor.keyword\",\"minorUsageId\" : \"domainObject.propertyDetails.usageCategoryMinor.keyword\", \"tenantId\" : \"dataObject.tenantId\", \"district\" : \"dataObject.tenantData.cityDistrictCode\" }", + "dateRefField": "dataObject.Bill.billDetails.receiptDate", + "indexName": "dss-collection_v1", + "aggrQuery": "{\"aggs\":{\"AGGR\":{\"filter\":{\"bool\":{\"must_not\":[{\"term\":{\"dataObject.tenantId.keyword\":\"pb.testing\"}}],\"must\":[{\"term\":{\"dataObject.Bill.billDetails.status.keyword\":\"Approved\"}}]}},\"aggs\":{\"Total Amount\":{\"sum\":{\"field\":\"dataObject.Bill.billDetails.taxAndPayments.taxAmount\"}},\"PT Tax\":{\"filter\":{\"term\":{\"dataObject.Bill.billDetails.billAccountDetails.taxHeadCode.keyword\":\"PT_TAX\"}},\"aggs\":{\"amount\":{\"sum\":{\"field\":\"dataObject.Bill.billDetails.billAccountDetails.taxHeadCode.amount\"}}}},\"Fire Cess\":{\"filter\":{\"term\":{\"dataObject.Bill.billDetails.billAccountDetails.taxHeadCode.keyword\":\"PT_FIRE_CESS\"}},\"aggs\":{\"amount\":{\"sum\":{\"field\":\"dataObject.Bill.billDetails.billAccountDetails.amount\"}}}},\"Cancer Cess\":{\"filter\":{\"term\":{\"dataObject.Bill.billDetails.billAccountDetails.taxHeadCode.keyword\":\"PT_CANCER_CESS\"}},\"aggs\":{\"amount\":{\"sum\":{\"field\":\"dataObject.Bill.billDetails.billAccountDetails.amount\"}}}},\"Rebate\":{\"filter\":{\"term\":{\"dataObject.Bill.billDetails.billAccountDetails.taxHeadCode.keyword\":\"PT_TIME_REBATE\"}},\"aggs\":{\"amount\":{\"sum\":{\"field\":\"dataObject.Bill.billDetails.billAccountDetails.amount\"}}}},\"Penalty\":{\"filter\":{\"term\":{\"dataObject.Bill.billDetails.billAccountDetails.taxHeadCode.keyword\":\"PT_TIME_PENALTY\"}},\"aggs\":{\"amount\":{\"sum\":{\"field\":\"dataObject.Bill.billDetails.billAccountDetails.amount\"}}}}}}}}" + } + ], + "isMdmsEnabled": true, + "filterKeys": [ + {"key": "tenantId", "column": "DDRs"} + + ], + "chartType": "table", + "valueType": "number", + "drillChart": "taxHeadsBreakupRevenue", + "isPostResponseHandler": true, + "documentType": "_doc", + "action": "", + "plotLabel": "DDRs", + "aggregationPaths": [ + "Total Amount", + "PT Tax", + "Fire Cess", + "Cancer Cess", + "Rebate", + "Penalty" + ], + "pathDataTypeMapping": [ + { + "Total Amount": "amount" + }, + { + "PT Tax": "amount" + }, + { + "Fire Cess": "amount" + }, + { + "Cancer Cess": "amount" + }, + { + "Rebate": "amount" + }, + { + "Penalty": "amount" + } + ], + "insight": { + }, + "_comment": "" + }, + "taxHeadsBreakupRevenue": { + "chartName": "DSS_PT_TAX_HEAD_BREAKUP_REVENUE", + "queries": [ + { + "module": "COMMON", + "requestQueryMap": "{\"module\" : \"dataObject.Bill.billDetails.businessService.keyword\", \"usageId\" : \"domainObject.propertyDetails.usageCategoryMajor.keyword\",\"minorUsageId\" : \"domainObject.propertyDetails.usageCategoryMinor.keyword\", \"tenantId\" : \"dataObject.tenantId\", \"district\" : \"dataObject.tenantData.cityDistrictCode\" }", + "dateRefField": "dataObject.Bill.billDetails.receiptDate", + "indexName": "dss-collection_v1", + "aggrQuery": "{\"aggs\":{\"AGGR\":{\"filter\":{\"bool\":{\"must_not\":[{\"term\":{\"dataObject.tenantId.keyword\":\"pb.testing\"}}],\"must\":[{\"term\":{\"dataObject.Bill.billDetails.status.keyword\":\"Approved\"}}]}},\"aggs\":{\"ULBs \":{\"terms\":{\"field\":\"dataObject.tenantId.keyword\",\"size\":200},\"aggs\":{\"Total Amount\":{\"sum\":{\"field\":\"dataObject.Bill.billDetails.taxAndPayments.taxAmount\"}},\"PT Tax\":{\"filter\":{\"term\":{\"dataObject.Bill.billDetails.billAccountDetails.taxHeadCode.keyword\":\"PT_TAX\"}},\"aggs\":{\"amount\":{\"sum\":{\"field\":\"dataObject.Bill.billDetails.billAccountDetails.taxHeadCode.amount\"}}}},\"Fire Cess\":{\"filter\":{\"term\":{\"dataObject.Bill.billDetails.billAccountDetails.taxHeadCode.keyword\":\"PT_FIRE_CESS\"}},\"aggs\":{\"amount\":{\"sum\":{\"field\":\"dataObject.Bill.billDetails.billAccountDetails.amount\"}}}},\"Cancer Cess\":{\"filter\":{\"term\":{\"dataObject.Bill.billDetails.billAccountDetails.taxHeadCode.keyword\":\"PT_CANCER_CESS\"}},\"aggs\":{\"amount\":{\"sum\":{\"field\":\"dataObject.Bill.billDetails.billAccountDetails.amount\"}}}},\"Rebate\":{\"filter\":{\"term\":{\"dataObject.Bill.billDetails.billAccountDetails.taxHeadCode.keyword\":\"PT_TIME_REBATE\"}},\"aggs\":{\"amount\":{\"sum\":{\"field\":\"dataObject.Bill.billDetails.billAccountDetails.amount\"}}}},\"Penalty\":{\"filter\":{\"term\":{\"dataObject.Bill.billDetails.billAccountDetails.taxHeadCode.keyword\":\"PT_TIME_PENALTY\"}},\"aggs\":{\"amount\":{\"sum\":{\"field\":\"dataObject.Bill.billDetails.billAccountDetails.amount\"}}}}}}}}}}" + } + ], + "filterKeys": [ + {"key": "tenantId", "column": "Boundary"} + + ], + "chartType": "table", + "valueType": "number", + "drillChart": "taxHeadsBreakupBoundaryDrill", + "drillFields": [ + "Ward", + "" + ], + "documentType": "_doc", + "action": "", + "plotLabel": "Boundary", + "aggregationPaths": [ + "Total Amount", + "PT Tax", + "Fire Cess", + "Cancer Cess", + "Rebate", + "Penalty" + ], + "pathDataTypeMapping": [ + { + "Total Amount": "amount" + }, + { + "PT Tax": "amount" + }, + { + "Fire Cess": "amount" + }, + { + "Cancer Cess": "amount" + }, + { + "Rebate": "amount" + }, + { + "Penalty": "amount" + } + ], + "insight": { + }, + "_comment": "" + }, + "taxHeadsBreakupBoundaryDrill": { + "chartName": "Tax head Breakup boundary", + "queries": [ + { + "module": "COMMON", + "requestQueryMap": "{\"module\" : \"dataObject.Bill.billDetails.businessService.keyword\", \"usageId\" : \"domainObject.propertyDetails.usageCategoryMajor.keyword\",\"minorUsageId\" : \"domainObject.propertyDetails.usageCategoryMinor.keyword\", \"tenantId\" : \"dataObject.tenantId\", \"district\" : \"dataObject.tenantData.cityDistrictCode\" }", + "dateRefField": "dataObject.Bill.billDetails.receiptDate", + "indexName": "dss-collection_v1", + "aggrQuery": "{\"aggs\":{\"AGGR\":{\"filter\":{\"bool\":{\"must_not\":[{\"term\":{\"dataObject.tenantId.keyword\":\"pb.testing\"}}],\"must\":[{\"term\":{\"dataObject.Bill.billDetails.status.keyword\":\"Approved\"}}]}},\"aggs\":{\"Ward \":{\"terms\":{\"field\":\"domainObject.ward.name.keyword\",\"size\":200},\"aggs\":{\"Total Amount\":{\"sum\":{\"field\":\"dataObject.Bill.billDetails.taxAndPayments.taxAmount\"}},\"PT Tax\":{\"filter\":{\"term\":{\"dataObject.Bill.billDetails.billAccountDetails.taxHeadCode.keyword\":\"PT_TAX\"}},\"aggs\":{\"amount\":{\"sum\":{\"field\":\"dataObject.Bill.billDetails.billAccountDetails.taxHeadCode.amount\"}}}},\"Fire Cess\":{\"filter\":{\"term\":{\"dataObject.Bill.billDetails.billAccountDetails.taxHeadCode.keyword\":\"PT_FIRE_CESS\"}},\"aggs\":{\"amount\":{\"sum\":{\"field\":\"dataObject.Bill.billDetails.billAccountDetails.amount\"}}}},\"Cancer Cess\":{\"filter\":{\"term\":{\"dataObject.Bill.billDetails.billAccountDetails.taxHeadCode.keyword\":\"PT_CANCER_CESS\"}},\"aggs\":{\"amount\":{\"sum\":{\"field\":\"dataObject.Bill.billDetails.billAccountDetails.amount\"}}}},\"Rebate\":{\"filter\":{\"term\":{\"dataObject.Bill.billDetails.billAccountDetails.taxHeadCode.keyword\":\"PT_TIME_REBATE\"}},\"aggs\":{\"amount\":{\"sum\":{\"field\":\"dataObject.Bill.billDetails.billAccountDetails.amount\"}}}},\"Penalty\":{\"filter\":{\"term\":{\"dataObject.Bill.billDetails.billAccountDetails.taxHeadCode.keyword\":\"PT_TIME_PENALTY\"}},\"aggs\":{\"amount\":{\"sum\":{\"field\":\"dataObject.Bill.billDetails.billAccountDetails.amount\"}}}}}}}}}}" + } + ], + "filterKeys": [ + {"key": "tenantId", "column": "Boundary"} + ], + "chartType": "table", + "valueType": "number", + "drillChart": "boundaryDrillDown", + "drillFields": [ + "Ward", + "" + ], + "documentType": "_doc", + "action": "", + "plotLabel": "Boundary", + "aggregationPaths": [ + "Total Amount", + "PT Tax", + "Fire Cess", + "Cancer Cess", + "Rebate", + "Penalty" + ], + "pathDataTypeMapping": [ + { + "Total Amount": "amount" + }, + { + "PT Tax": "amount" + }, + { + "Fire Cess": "amount" + }, + { + "Cancer Cess": "amount" + }, + { + "Rebate": "amount" + }, + { + "Penalty": "amount" + } + ], + "insight": { + }, + "_comment": "" + }, + "taxHeadsBreakupUsage": { + "chartName": "DSS_PT_TAX_HEAD_BREAKUP_USAGE", + "queries": [ + { + "module": "COMMON", + "requestQueryMap": "{\"module\" : \"dataObject.Bill.billDetails.businessService.keyword\", \"tenantId\" : \"dataObject.tenantId\", \"district\" : \"dataObject.tenantData.cityDistrictCode\" }", + "dateRefField": "dataObject.Bill.billDetails.receiptDate", + "indexName": "dss-collection_v1", + "aggrQuery": "{\"aggs\":{\"AGGR\":{\"filter\":{\"bool\":{\"must_not\":[{\"term\":{\"dataObject.tenantId.keyword\":\"pb.testing\"}}],\"must\":[{\"term\":{\"dataObject.Bill.billDetails.status.keyword\":\"Approved\"}}]}},\"aggs\":{\"Usage \":{\"terms\":{\"field\":\"domainObject.propertyDetails.units.usageCategoryMajor.keyword\",\"size\":200},\"aggs\":{\"Total Amount\":{\"sum\":{\"field\":\"dataObject.Bill.billDetails.taxAndPayments.taxAmount\"}},\"PT Tax\":{\"filter\":{\"term\":{\"dataObject.Bill.billDetails.billAccountDetails.taxHeadCode.keyword\":\"PT_TAX\"}},\"aggs\":{\"amount\":{\"sum\":{\"field\":\"dataObject.Bill.billDetails.billAccountDetails.taxHeadCode.amount\"}}}},\"Fire Cess\":{\"filter\":{\"term\":{\"dataObject.Bill.billDetails.billAccountDetails.taxHeadCode.keyword\":\"PT_FIRE_CESS\"}},\"aggs\":{\"amount\":{\"sum\":{\"field\":\"dataObject.Bill.billDetails.billAccountDetails.amount\"}}}},\"Cancer Cess\":{\"filter\":{\"term\":{\"dataObject.Bill.billDetails.billAccountDetails.taxHeadCode.keyword\":\"PT_CANCER_CESS\"}},\"aggs\":{\"amount\":{\"sum\":{\"field\":\"dataObject.Bill.billDetails.billAccountDetails.amount\"}}}},\"Rebate\":{\"filter\":{\"term\":{\"dataObject.Bill.billDetails.billAccountDetails.taxHeadCode.keyword\":\"PT_TIME_REBATE\"}},\"aggs\":{\"amount\":{\"sum\":{\"field\":\"dataObject.Bill.billDetails.billAccountDetails.amount\"}}}},\"Penalty\":{\"filter\":{\"term\":{\"dataObject.Bill.billDetails.billAccountDetails.taxHeadCode.keyword\":\"PT_TIME_PENALTY\"}},\"aggs\":{\"amount\":{\"sum\":{\"field\":\"dataObject.Bill.billDetails.billAccountDetails.amount\"}}}}}}}}}}" + } + ], + "filterKeys": [ + {"key": "usageId", "column": "Usage Type"} + ], + "chartType": "table", + "valueType": "number", + "drillChart": "taxHeadsBreakupUsageMinor", + "documentType": "_doc", + "action": "", + "plotLabel": "Usage Type", + "aggregationPaths": [ + "Total Amount", + "PT Tax", + "Fire Cess", + "Cancer Cess", + "Rebate", + "Penalty" + ], + "pathDataTypeMapping": [ + { + "Total Amount": "amount" + }, + { + "PT Tax": "amount" + }, + { + "Fire Cess": "amount" + }, + { + "Cancer Cess": "amount" + }, + { + "Rebate": "amount" + }, + { + "Penalty": "amount" + } + ], + "insight": { + }, + "_comment": "" + }, + "taxHeadsBreakupUsageMinor": { + "chartName": "", + "queries": [ + { + "module": "COMMON", + "requestQueryMap": "{\"usageId\" : \"domainObject.propertyDetails.usageCategoryMajor.keyword\",\"minorUsageId\" : \"domainObject.propertyDetails.usageCategoryMinor.keyword\",\"module\" : \"dataObject.Bill.billDetails.businessService.keyword\", \"tenantId\" : \"dataObject.tenantId\", \"district\" : \"dataObject.tenantData.cityDistrictCode\" }", + "dateRefField": "dataObject.Bill.billDetails.receiptDate", + "indexName": "dss-collection_v1", + "aggrQuery": "{\"aggs\":{\"AGGR\":{\"filter\":{\"bool\":{\"must_not\":[{\"term\":{\"dataObject.tenantId.keyword\":\"pb.testing\"}}],\"must\":[{\"term\":{\"dataObject.Bill.billDetails.status.keyword\":\"Approved\"}}]}},\"aggs\":{\"Usage \":{\"terms\":{\"field\":\"domainObject.propertyDetails.usageCategoryMinor.keyword\",\"size\":200},\"aggs\":{\"Total Amount\":{\"sum\":{\"field\":\"dataObject.Bill.billDetails.taxAndPayments.taxAmount\"}},\"PT Tax\":{\"filter\":{\"term\":{\"dataObject.Bill.billDetails.billAccountDetails.taxHeadCode.keyword\":\"PT_TAX\"}},\"aggs\":{\"amount\":{\"sum\":{\"field\":\"dataObject.Bill.billDetails.billAccountDetails.taxHeadCode.amount\"}}}},\"Fire Cess\":{\"filter\":{\"term\":{\"dataObject.Bill.billDetails.billAccountDetails.taxHeadCode.keyword\":\"PT_FIRE_CESS\"}},\"aggs\":{\"amount\":{\"sum\":{\"field\":\"dataObject.Bill.billDetails.billAccountDetails.amount\"}}}},\"Cancer Cess\":{\"filter\":{\"term\":{\"dataObject.Bill.billDetails.billAccountDetails.taxHeadCode.keyword\":\"PT_CANCER_CESS\"}},\"aggs\":{\"amount\":{\"sum\":{\"field\":\"dataObject.Bill.billDetails.billAccountDetails.amount\"}}}},\"Rebate\":{\"filter\":{\"term\":{\"dataObject.Bill.billDetails.billAccountDetails.taxHeadCode.keyword\":\"PT_TIME_REBATE\"}},\"aggs\":{\"amount\":{\"sum\":{\"field\":\"dataObject.Bill.billDetails.billAccountDetails.amount\"}}}},\"Penalty\":{\"filter\":{\"term\":{\"dataObject.Bill.billDetails.billAccountDetails.taxHeadCode.keyword\":\"PT_TIME_PENALTY\"}},\"aggs\":{\"amount\":{\"sum\":{\"field\":\"dataObject.Bill.billDetails.billAccountDetails.amount\"}}}}}}}}}}" + } + ], + "filterKeys": [ + {"key": "minorUsageId", "column": "Minor Usage Type"} + ], + "chartType": "table", + "valueType": "number", + "drillChart": "taxHeadsBreakupUsageSubMinor", + "documentType": "_doc", + "action": "", + "plotLabel": "Minor Usage Type", + "aggregationPaths": [ + "Total Amount", + "PT Tax", + "Fire Cess", + "Cancer Cess", + "Rebate", + "Penalty" + ], + "pathDataTypeMapping": [ + { + "Total Amount": "amount" + }, + { + "PT Tax": "amount" + }, + { + "Fire Cess": "amount" + }, + { + "Cancer Cess": "amount" + }, + { + "Rebate": "amount" + }, + { + "Penalty": "amount" + } + ], + "insight": { + }, + "_comment": "" + }, + "taxHeadsBreakupUsageSubMinor": { + "chartName": "", + "queries": [ + { + "module": "COMMON", + "requestQueryMap": "{\"usageId\" : \"domainObject.propertyDetails.usageCategoryMajor.keyword\",\"minorUsageId\" : \"domainObject.propertyDetails.usageCategoryMinor.keyword\",\"module\" : \"dataObject.Bill.billDetails.businessService.keyword\", \"tenantId\" : \"dataObject.tenantId\", \"district\" : \"dataObject.tenantData.cityDistrictCode\" }", + "dateRefField": "dataObject.Bill.billDetails.receiptDate", + "indexName": "dss-collection_v1", + "aggrQuery": "{\"aggs\":{\"AGGR\":{\"filter\":{\"bool\":{\"must_not\":[{\"term\":{\"dataObject.tenantId.keyword\":\"pb.testing\"}}],\"must\":[{\"term\":{\"dataObject.Bill.billDetails.status.keyword\":\"Approved\"}}]}},\"aggs\":{\"Usage \":{\"terms\":{\"field\":\"domainObject.propertyDetails.units.usageCategorySubMinor.keyword\",\"size\":200},\"aggs\":{\"Total Amount\":{\"sum\":{\"field\":\"dataObject.Bill.billDetails.taxAndPayments.taxAmount\"}},\"PT Tax\":{\"filter\":{\"term\":{\"dataObject.Bill.billDetails.billAccountDetails.taxHeadCode.keyword\":\"PT_TAX\"}},\"aggs\":{\"amount\":{\"sum\":{\"field\":\"dataObject.Bill.billDetails.billAccountDetails.taxHeadCode.amount\"}}}},\"Fire Cess\":{\"filter\":{\"term\":{\"dataObject.Bill.billDetails.billAccountDetails.taxHeadCode.keyword\":\"PT_FIRE_CESS\"}},\"aggs\":{\"amount\":{\"sum\":{\"field\":\"dataObject.Bill.billDetails.billAccountDetails.amount\"}}}},\"Cancer Cess\":{\"filter\":{\"term\":{\"dataObject.Bill.billDetails.billAccountDetails.taxHeadCode.keyword\":\"PT_CANCER_CESS\"}},\"aggs\":{\"amount\":{\"sum\":{\"field\":\"dataObject.Bill.billDetails.billAccountDetails.amount\"}}}},\"Rebate\":{\"filter\":{\"term\":{\"dataObject.Bill.billDetails.billAccountDetails.taxHeadCode.keyword\":\"PT_TIME_REBATE\"}},\"aggs\":{\"amount\":{\"sum\":{\"field\":\"dataObject.Bill.billDetails.billAccountDetails.amount\"}}}},\"Penalty\":{\"filter\":{\"term\":{\"dataObject.Bill.billDetails.billAccountDetails.taxHeadCode.keyword\":\"PT_TIME_PENALTY\"}},\"aggs\":{\"amount\":{\"sum\":{\"field\":\"dataObject.Bill.billDetails.billAccountDetails.amount\"}}}}}}}}}}" + } + ], + "filterKeys": [ + "" + ], + "chartType": "table", + "valueType": "number", + "drillChart": "", + "documentType": "_doc", + "action": "", + "plotLabel": "Sub Minor Usage Type", + "aggregationPaths": [ + "Total Amount", + "PT Tax", + "Fire Cess", + "Cancer Cess", + "Rebate", + "Penalty" + ], + "pathDataTypeMapping": [ + { + "Total Amount": "amount" + }, + { + "PT Tax": "amount" + }, + { + "Fire Cess": "amount" + }, + { + "Cancer Cess": "amount" + }, + { + "Rebate": "amount" + }, + { + "Penalty": "amount" + } + ], + "insight": { + }, + "_comment": "" + }, + "_comment": "PT COLLECTION CHARTS (Demands) ", + "demandCollectionIndexDDRRevenue": { + "chartName": "DSS_PT_DEMAND_COLLECTION_BOUNDARY", + "queries": [ + { + "module": "COMMON", + "requestQueryMap": "{\"module\" : \"dataObject.Bill.billDetails.businessService.keyword\", \"usageId\" : \"domainObject.propertyDetails.usageCategoryMajor.keyword\",\"minorUsageId\" : \"domainObject.propertyDetails.usageCategoryMinor.keyword\", \"tenantId\" : \"dataObject.tenantId\", \"district\" : \"dataObject.tenantData.cityDistrictCode\" }", + "dateRefField": "dataObject.Bill.billDetails.receiptDate", + "indexName": "dss-collection_v1", + "aggrQuery": "{\"aggs\":{\"AGGR\":{\"filter\":{\"bool\":{\"must_not\":[{\"term\":{\"dataObject.tenantId.keyword\":\"pb.testing\"}}],\"must\":[{\"term\":{\"dataObject.Bill.billDetails.status.keyword\":\"Approved\"}}]}},\"aggs\":{\"Total Collection\":{\"sum\":{\"field\":\"dataObject.Bill.billDetails.amountPaid\"}},\"Transactions\":{\"value_count\":{\"field\":\"dataObject.transactionId.keyword\"}},\"Assessed Properties\":{\"value_count\":{\"field\":\"domainObject.propertyDetails.assessmentNumber.keyword\"}}}}}}" + }, + { + "module": "COMMON", + "requestQueryMap": "{\"module\" : \"businessService.keyword\", \"tenantId\" : \"tenantIdForMunicipalCorporation.keyword\"}", + "dateRefField": "", + "indexName": "dss-target_v1", + "aggrQuery": "{\"aggs\":{\"Target Collection\":{\"sum\":{\"field\":\"budgetProposedForMunicipalCorporation\"}}}}}}" + } + ], + "isMdmsEnabled": true, + "filterKeys": [ + {"key": "tenantId", "column": "DDRs"} + ], + "chartType": "table", + "valueType": "number", + "drillChart": "demandCollectionIndexBoundaryRevenue", + "action": "", + "plotLabel": "DDRs", + "isPostResponseHandler": true, + "postAggregationTheory" : "repsonseToDifferenceOfDates", + "aggregationPaths": [ + "Total Collection", + "Transactions", + "Assessed Properties", + "Target Collection" + ], + "pathDataTypeMapping": [ + { + "Total Collection": "amount" + }, + { + "Transactions": "number" + }, + { + "Assessed Properties": "number" + }, + { + "Target Collection": "amount" + } + ], + "insight": { + }, + "_comment": "" + }, + "demandCollectionIndexBoundaryRevenue": { + "chartName": "DSS_PT_DEMAND_COLLECTION_BOUNDARY", + "queries": [ + { + "module": "COMMON", + "requestQueryMap": "{\"module\" : \"dataObject.Bill.billDetails.businessService.keyword\", \"usageId\" : \"domainObject.propertyDetails.usageCategoryMajor.keyword\",\"minorUsageId\" : \"domainObject.propertyDetails.usageCategoryMinor.keyword\", \"tenantId\" : \"dataObject.tenantId\", \"district\" : \"dataObject.tenantData.cityDistrictCode\" }", + "dateRefField": "dataObject.Bill.billDetails.receiptDate", + "indexName": "dss-collection_v1", + "aggrQuery": "{\"aggs\":{\"AGGR\":{\"filter\":{\"bool\":{\"must_not\":[{\"term\":{\"dataObject.tenantId.keyword\":\"pb.testing\"}}],\"must\":[{\"term\":{\"dataObject.Bill.billDetails.status.keyword\":\"Approved\"}}]}},\"aggs\":{\"ULBs \":{\"terms\":{\"field\":\"domainObject.tenantId.keyword\",\"size\":200},\"aggs\":{\"Total Collection\":{\"sum\":{\"field\":\"dataObject.Bill.billDetails.amountPaid\"}},\"Transactions\":{\"value_count\":{\"field\":\"dataObject.transactionId.keyword\"}},\"Assessed Properties\":{\"value_count\":{\"field\":\"domainObject.propertyDetails.assessmentNumber.keyword\"}}}}}}}}" + }, + { + "module": "COMMON", + "requestQueryMap": "{\"module\" : \"businessService.keyword\", \"tenantId\" : \"tenantIdForMunicipalCorporation.keyword\"}", + "dateRefField": "", + "indexName": "dss-target_v1", + "aggrQuery": "{\"aggs\":{\"ULBs \":{\"terms\":{\"field\":\"tenantIdForMunicipalCorporation.keyword\",\"size\":1000},\"aggs\":{\"Target Collection\":{\"sum\":{\"field\":\"budgetProposedForMunicipalCorporation\"}}}}}}" + } + ], + "filterKeys": [ + {"key": "tenantId", "column": "Boundary"} + ], + "postAggregationTheory" : "repsonseToDifferenceOfDates", "chartType": "table", + "valueType": "number", + "drillChart": "boundaryDrillDown", + "drillFields": [ + "Ward", + "" + ], + "documentType": "_doc", + "action": "", + "plotLabel": "Boundary", + "aggregationPaths": [ + "Total Collection", + "Transactions", + "Assessed Properties", + "Target Collection" + ], + "pathDataTypeMapping": [ + { + "Total Collection": "amount" + }, + { + "Transactions": "number" + }, + { + "Assessed Properties": "number" + }, + { + "Target Collection": "amount" + } + ], + "insight": { + }, + "_comment": "" + }, + "boundaryDrillDown": { + "kind": "drillDown", + "chartName": "Boundary", + "queries": [ + { + "module": "COMMON", + "requestQueryMap": "{\"module\" : \"dataObject.Bill.billDetails.businessService.keyword\", \"usageId\" : \"domainObject.propertyDetails.usageCategoryMajor.keyword\",\"minorUsageId\" : \"domainObject.propertyDetails.usageCategoryMinor.keyword\",\"tenantId\" : \"dataObject.tenantId\", \"district\" : \"dataObject.tenantData.cityDistrictCode\" }", + "dateRefField": "dataObject.Bill.billDetails.receiptDate", + "indexName": "dss-collection_v1", + "aggrQuery": "{\"aggs\":{\"AGGR\":{\"filter\":{\"bool\":{\"must_not\":[{\"term\":{\"dataObject.tenantId.keyword\":\"pb.testing\"}}],\"must\":[{\"term\":{\"dataObject.Bill.billDetails.status.keyword\":\"Approved\"}}]}},\"aggs\":{\"Ward \":{\"terms\":{\"field\":\"domainObject.ward.name.keyword\",\"size\":200},\"aggs\":{\"Total Collection\":{\"sum\":{\"field\":\"dataObject.Bill.billDetails.amountPaid\"}},\"Transactions\":{\"value_count\":{\"field\":\"dataObject.transactionId.keyword\"}},\"Assessed Properties\":{\"value_count\":{\"field\":\"domainObject.propertyDetails.assessmentNumber.keyword\"}}}}}}}}" + } + ], + "chartType": "table", + "valueType": "number", + "drillChart": "none", + "documentType": "_doc", + "action": "", + "plotLabel": "Ward", + "aggregationPaths": [ + "Total Collection", + "Transactions", + "Assessed Properties", + "Target Collection" + ], + "pathDataTypeMapping": [ + { + "Total Collection": "amount" + }, + { + "Transactions": "number" + }, + { + "Assessed Properties": "number" + }, + { + "Target Collection": "amount" + } + ], + "insight": { + }, + "_comment": "" + }, + "demandCollectionIndexUsageRevenue": { + "chartName": "DSS_PT_DEMAND_COLLECTION_USAGETYPE", + "queries": [ + { + "module": "COMMON", + "requestQueryMap": "{\"module\" : \"dataObject.Bill.billDetails.businessService.keyword\",\"usageId\" : \"domainObject.propertyDetails.usageCategoryMajor.keyword\",\"minorUsageId\" : \"domainObject.propertyDetails.usageCategoryMinor.keyword\", \"tenantId\" : \"dataObject.tenantId\", \"district\" : \"dataObject.tenantData.cityDistrictCode\" }", + "dateRefField": "dataObject.Bill.billDetails.receiptDate", + "indexName": "dss-collection_v1", + "aggrQuery": "{\"aggs\":{\"AGGR\":{\"filter\":{\"bool\":{\"must_not\":[{\"term\":{\"dataObject.tenantId.keyword\":\"pb.testing\"}}],\"must\":[{\"term\":{\"dataObject.Bill.billDetails.status.keyword\":\"Approved\"}}]}},\"aggs\":{\"UsageType \":{\"terms\":{\"field\":\"domainObject.propertyDetails.units.usageCategoryMajor.keyword\",\"size\":200},\"aggs\":{\"Total Collection\":{\"sum\":{\"field\":\"dataObject.Bill.billDetails.amountPaid\"}},\"Transactions\":{\"value_count\":{\"field\":\"dataObject.transactionId.keyword\"}},\"Assessed Properties\":{\"value_count\":{\"field\":\"domainObject.propertyDetails.assessmentNumber.keyword\"}}}}}}}}" + } + ], + "filterKeys": [ + {"key": "usageId", "column": "Usage Type"} + ], + "chartType": "table", + "valueType": "number", + "drillChart": "usageRevenueDrillDownMinor", + "drillFields": [ + "Ward", + "" + ], + "documentType": "_doc", + "action": "", + "plotLabel": "Usage Type", + "aggregationPaths": [ + "Total Collection", + "Transactions", + "Assessed Properties", + "Target Collection" + ], + "pathDataTypeMapping": [ + { + "Total Collection": "amount" + }, + { + "Transactions": "number" + }, + { + "Assessed Properties": "number" + }, + { + "Target Collection": "amount" + } + ], + "insight": { + }, + "_comment": "" + }, + "usageRevenueDrillDownMinor": { + "chartName": "", + "queries": [ + { + "module": "COMMON", + "requestQueryMap": "{\"usageId\" : \"domainObject.propertyDetails.usageCategoryMajor.keyword\",\"minorUsageId\" : \"domainObject.propertyDetails.usageCategoryMinor.keyword\",\"module\" : \"dataObject.Bill.billDetails.businessService.keyword\", \"tenantId\" : \"dataObject.tenantId\", \"district\" : \"dataObject.tenantData.cityDistrictCode\" }", + "dateRefField": "dataObject.Bill.billDetails.receiptDate", + "indexName": "dss-collection_v1", + "aggrQuery": "{\"aggs\":{\"AGGR\":{\"filter\":{\"bool\":{\"must_not\":[{\"term\":{\"dataObject.tenantId.keyword\":\"pb.testing\"}}],\"must\":[{\"term\":{\"dataObject.Bill.billDetails.status.keyword\":\"Approved\"}}]}},\"aggs\":{\"UsageType \":{\"terms\":{\"field\":\"domainObject.propertyDetails.usageCategoryMinor.keyword\",\"size\":200},\"aggs\":{\"Total Collection\":{\"sum\":{\"field\":\"dataObject.Bill.billDetails.amountPaid\"}},\"Transactions\":{\"value_count\":{\"field\":\"dataObject.transactionId.keyword\"}},\"Assessed Properties\":{\"value_count\":{\"field\":\"domainObject.propertyDetails.assessmentNumber.keyword\"}}}}}}}}" + } + ], + "filterKeys": [ + {"key": "minorUsageId", "column": "Minor Usage Type"} + ], + "chartType": "table", + "valueType": "number", + "drillChart": "usageRevenueDrillDownSubMinor", + "documentType": "_doc", + "action": "", + "plotLabel": "Minor Usage Type", + "aggregationPaths": [ + "Total Collection", + "Transactions", + "Assessed Properties" + ], + "pathDataTypeMapping": [ + { + "Total Collection": "amount" + }, + { + "Transactions": "number" + }, + { + "Assessed Properties": "number" + } + ], + "insight": { + }, + "_comment": "" + }, + "usageRevenueDrillDownSubMinor": { + "chartName": "", + "queries": [ + { + "module": "COMMON", + "requestQueryMap": "{\"usageId\" : \"domainObject.propertyDetails.usageCategoryMajor.keyword\",\"minorUsageId\" : \"domainObject.propertyDetails.usageCategoryMinor.keyword\",\"module\" : \"dataObject.Bill.billDetails.businessService.keyword\", \"tenantId\" : \"dataObject.tenantId\", \"district\" : \"dataObject.tenantData.cityDistrictCode\" }", + "dateRefField": "dataObject.Bill.billDetails.receiptDate", + "indexName": "dss-collection_v1", + "aggrQuery": "{\"aggs\":{\"AGGR\":{\"filter\":{\"bool\":{\"must_not\":[{\"term\":{\"dataObject.tenantId.keyword\":\"pb.testing\"}}],\"must\":[{\"term\":{\"dataObject.Bill.billDetails.status.keyword\":\"Approved\"}}]}},\"aggs\":{\"UsageType \":{\"terms\":{\"field\":\"domainObject.propertyDetails.units.usageCategorySubMinor.keyword\",\"size\":200},\"aggs\":{\"Total Collection\":{\"sum\":{\"field\":\"dataObject.Bill.billDetails.amountPaid\"}},\"Transactions\":{\"value_count\":{\"field\":\"dataObject.transactionId.keyword\"}},\"Assessed Properties\":{\"value_count\":{\"field\":\"domainObject.propertyDetails.assessmentNumber.keyword\"}}}}}}}}" + } + ], + "filterKeys": [ + + ], + "chartType": "table", + "valueType": "number", + "drillChart": "none", + "documentType": "_doc", + "action": "", + "plotLabel": "Sub Minor Usage Type", + "aggregationPaths": [ + "Total Collection", + "Transactions", + "Assessed Properties" + ], + "pathDataTypeMapping": [ + { + "Total Collection": "amount" + }, + { + "Transactions": "number" + }, + { + "Assessed Properties": "number" + } + ], + "insight": { + }, + "_comment": "" + }, + + + + "_comment": "TL COLLECTION CHARTS", + "licenseByLicenseType": { + "chartName": "DSS_TL_LICENSE_BY_TYPE", + "queries": [ + { + "module": "COMMON", + "requestQueryMap": "{\"module\" : \"Data.Bill.billDetails.businessService.keyword\", \"tenantId\" : \"Data.tenantData.code\", \"district\" : \"Data.tenantData.cityDistrictCode\"}", + "dateRefField": "Data.Bill.billDetails.receiptDate", + "indexName": "collectionsindex-v1", + "aggrQuery": "{\"aggs\":{\"AGGR\":{\"filter\":{\"bool\":{\"must_not\":[{\"term\":{\"Data.tenantId.keyword\":\"pb.testing\"}}],\"must\":[{\"term\":{\"Data.Bill.billDetails.status.keyword\":\"Approved\"}}]}},\"aggs\":{\"License Instrument Type\":{\"terms\":{\"field\":\"Data.instrument.instrumentType.name.keyword\"},\"aggs\":{\"Licens Issued\":{\"sum\":{\"field\":\"Data.Bill.billDetails.amountPaid\"}}}}}}}}" + } + ], + "chartType": "pie", + "valueType": "number", + "action": "", + "documentType": "_doc", + "drillChart": "none", + "aggregationPaths": [ + "License Instrument Type" + ], + "insight": { + }, + "_comment": " License Application which are issued by License Type" + }, + "licenseIssuedDDRRevenue": { + "chartName": "DSS_TL_DEMAND_COLLECTION_BOUNDARY", + "queries": [ + { + "module": "COMMON", + "requestQueryMap": "{\"module\" : \"dataObject.Bill.billDetails.businessService.keyword\", \"tenantId\" : \"dataObject.tenantId\", \"district\" : \"dataObject.tenantData.cityDistrictCode\" }", + "dateRefField": "dataObject.Bill.billDetails.receiptDate", + "indexName": "dss-collection_v1", + "aggrQuery": "{\"aggs\":{\"AGGR\":{\"filter\":{\"bool\":{\"must_not\":[{\"term\":{\"dataObject.tenantId.keyword\":\"pb.testing\"}}],\"must\":[{\"term\":{\"dataObject.Bill.billDetails.status.keyword\":\"Approved\"}}]}},\"aggs\":{\"Total Collection\":{\"sum\":{\"field\":\"dataObject.Bill.billDetails.amountPaid\"}},\"Transactions\":{\"value_count\":{\"field\":\"dataObject.transactionId.keyword\"}},\"Total Licence Issued\":{\"value_count\":{\"field\":\"domainObject.tradelicense.licensenumber.keyword\"}}}}}}" + }, + { + "module": "COMMON", + "requestQueryMap": "{\"module\" : \"businessService.keyword\", \"tenantId\" : \"tenantIdForMunicipalCorporation.keyword\"}", + "dateRefField": "", + "indexName": "dss-target_v1", + "aggrQuery": "{\"aggs\":{\"Target Collection\":{\"sum\":{\"field\":\"budgetProposedForMunicipalCorporation\"}}}}}}" + } + ], + "isMdmsEnabled": true, + "filterKeys": [ + {"key": "tenantId", "column": "DDRs"} + ], + "isPostResponseHandler": true, + "postAggregationTheory" : "repsonseToDifferenceOfDates", + "chartType": "table", + "valueType": "number", + "drillChart": "licenseIssuedBoundaryRevenue", + "documentType": "_doc", + "action": "", + "plotLabel": "DDRs", + "aggregationPaths": [ + "Total Collection", + "Transactions", + "Total Licence Issued", + "Target Collection" + ], + "pathDataTypeMapping": [ + { + "Total Collection": "amount" + }, + { + "Transactions": "number" + }, + { + "Total Licence Issued": "number" + }, + { + "Target Collection": "amount" + } + ], + "insight": { + }, + "_comment": "" + }, + "licenseIssuedBoundaryRevenue": { + "chartName": "DSS_TL_DEMAND_COLLECTION_BOUNDARY", + "queries": [ + { + "module": "COMMON", + "requestQueryMap": "{\"module\" : \"dataObject.Bill.billDetails.businessService.keyword\", \"tenantId\" : \"dataObject.tenantId\", \"district\" : \"dataObject.tenantData.cityDistrictCode\" }", + "dateRefField": "dataObject.Bill.billDetails.receiptDate", + "indexName": "dss-collection_v1", + "aggrQuery": "{\"aggs\":{\"AGGR\":{\"filter\":{\"bool\":{\"must_not\":[{\"term\":{\"dataObject.tenantId.keyword\":\"pb.testing\"}}],\"must\":[{\"term\":{\"dataObject.Bill.billDetails.status.keyword\":\"Approved\"}}]}},\"aggs\":{\"ULBs \":{\"terms\":{\"field\":\"dataObject.tenantId.keyword\",\"size\":200},\"aggs\":{\"Total Collection\":{\"sum\":{\"field\":\"dataObject.Bill.billDetails.amountPaid\"}},\"Transactions\":{\"value_count\":{\"field\":\"dataObject.transactionId.keyword\"}},\"Total Licence Issued\":{\"value_count\":{\"field\":\"domainObject.tradelicense.licensenumber.keyword\"}}}}}}}}" + }, + { + "module": "COMMON", + "requestQueryMap": "{\"module\" : \"businessService.keyword\", \"tenantId\" : \"tenantIdForMunicipalCorporation.keyword\"}", + "dateRefField": "", + "indexName": "dss-target_v1", + "aggrQuery": "{\"aggs\":{\"ULBs \":{\"terms\":{\"field\":\"tenantIdForMunicipalCorporation.keyword\",\"size\":1000},\"aggs\":{\"Target Collection\":{\"sum\":{\"field\":\"budgetProposedForMunicipalCorporation\"}}}}}}" + } + ], + "filterKeys": [ + {"key": "tenantId", "column": "Boundary"} + ], + "postAggregationTheory" : "repsonseToDifferenceOfDates", + "chartType": "table", + "valueType": "number", + "drillChart": "licenseIssuedBoundaryDrillDown", + "documentType": "_doc", + "action": "", + "plotLabel": "Boundary", + "aggregationPaths": [ + "Total Collection", + "Transactions", + "Total Licence Issued", + "Target Collection" + ], + "pathDataTypeMapping": [ + { + "Total Collection": "amount" + }, + { + "Transactions": "number" + }, + { + "Total Licence Issued": "number" + }, + { + "Target Collection": "amount" + } + ], + "insight": { + }, + "_comment": "" + }, + "licenseIssuedBoundaryDrillDown": { + "chartName": "", + "queries": [ + { + "module": "COMMON", + "requestQueryMap": "{\"module\" : \"dataObject.Bill.billDetails.businessService.keyword\", \"tenantId\" : \"dataObject.tenantId\", \"district\" : \"dataObject.tenantData.cityDistrictCode\" }", + "dateRefField": "dataObject.Bill.billDetails.receiptDate", + "indexName": "dss-collection_v1", + "aggrQuery": "{\"aggs\":{\"AGGR\":{\"filter\":{\"bool\":{\"must_not\":[{\"term\":{\"dataObject.tenantId.keyword\":\"pb.testing\"}}],\"must\":[{\"term\":{\"dataObject.Bill.billDetails.status.keyword\":\"Approved\"}}]}},\"aggs\":{\"Ward \":{\"terms\":{\"field\":\"domainObject.ward.name.keyword\",\"size\":200},\"aggs\":{\"Total Collection\":{\"sum\":{\"field\":\"dataObject.Bill.billDetails.amountPaid\"}},\"Transactions\":{\"value_count\":{\"field\":\"dataObject.transactionId.keyword\"}},\"Total Licence Issued\":{\"value_count\":{\"field\":\"domainObject.tradelicense.licensenumber.keyword\"}}}}}}}}" + } + ], + "postAggregationTheory" : "repsonseToDifferenceOfDates", + "chartType": "table", + "valueType": "number", + "drillChart": "none", + "documentType": "_doc", + "action": "", + "plotLabel": "Ward", + "aggregationPaths": [ + "Total Collection", + "Transactions", + "Total Licence Issued", + "Target Collection" + ], + "pathDataTypeMapping": [ + { + "Total Collection": "amount" + }, + { + "Transactions": "number" + }, + { + "Total Licence Issued": "number" + }, + { + "Target Collection": "amount" + } + ], + "insight": { + }, + "_comment": "" + }, + "licenceTaxHeadsBreakupDDR": { + "chartName": "DSS_TL_TAX_HEAD_BREAKUP_BOUNDARY", + "queries": [ + { + "module": "COMMON", + "requestQueryMap": "{\"module\" : \"dataObject.Bill.billDetails.businessService.keyword\", \"tenantId\" : \"dataObject.tenantId\", \"district\" : \"dataObject.tenantData.cityDistrictCode\" }", + "dateRefField": "dataObject.Bill.billDetails.receiptDate", + "indexName": "dss-collection_v1", + "aggrQuery": "{\"aggs\":{\"AGGR\":{\"filter\":{\"bool\":{\"must_not\":[{\"term\":{\"dataObject.tenantId.keyword\":\"pb.testing\"}}],\"must\":[{\"term\":{\"dataObject.Bill.billDetails.status.keyword\":\"Approved\"}}]}},\"aggs\":{\"Tax\":{\"filter\":{\"term\":{\"dataObject.Bill.billDetails.billAccountDetails.taxHeadCode.keyword\":\"TL_TAX\"}},\"aggs\":{\"amount\":{\"sum\":{\"field\":\"dataObject.Bill.billDetails.billAccountDetails.amount\"}}}},\"Adhoc Penalty\":{\"filter\":{\"term\":{\"dataObject.Bill.billDetails.billAccountDetails.taxHeadCode.keyword\":\"TL_ADHOC_PENALTY\"}},\"aggs\":{\"amount\":{\"sum\":{\"field\":\"dataObject.Bill.billDetails.billAccountDetails.amount\"}}}},\"Adhoc Rebate\":{\"filter\":{\"term\":{\"dataObject.Bill.billDetails.billAccountDetails.taxHeadCode.keyword\":\"TL_ADHOC_REBATE\"}},\"aggs\":{\"amount\":{\"sum\":{\"field\":\"dataObject.Bill.billDetails.billAccountDetails.amount\"}}}}}}}}" + } + ], + "isMdmsEnabled": true, + "filterKeys": [ + {"key": "tenantId", "column": "DDRs"} + ], + "isPostResponseHandler": true, + "chartType": "table", + "valueType": "number", + "drillChart": "licenceTaxHeadsBreakupBoundary", + "documentType": "_doc", + "action": "", + "plotLabel": "DDRs", + "aggregationPaths": [ + "Tax", + "Adhoc Penalty", + "Adhoc Rebate" + ], + "pathDataTypeMapping": [ + { + "Tax": "amount" + }, + { + "Adhoc Penalty": "amount" + }, + { + "Adhoc Rebate": "amount" + } + ], + "insight": { + }, + "_comment": "" + }, + "licenceTaxHeadsBreakupBoundary": { + "chartName": "DSS_TL_TAX_HEAD_BREAKUP_BOUNDARY", + "queries": [ + { + "module": "COMMON", + "requestQueryMap": "{\"module\" : \"dataObject.Bill.billDetails.businessService.keyword\", \"tenantId\" : \"dataObject.tenantId\", \"district\" : \"dataObject.tenantData.cityDistrictCode\" }", + "dateRefField": "dataObject.Bill.billDetails.receiptDate", + "indexName": "dss-collection_v1", + "aggrQuery": "{\"aggs\":{\"AGGR\":{\"filter\":{\"bool\":{\"must_not\":[{\"term\":{\"dataObject.tenantId.keyword\":\"pb.testing\"}}],\"must\":[{\"term\":{\"dataObject.Bill.billDetails.status.keyword\":\"Approved\"}}]}},\"aggs\":{\"ULBs \":{\"terms\":{\"field\":\"dataObject.tenantId.keyword\",\"size\":200},\"aggs\":{\"Tax\":{\"filter\":{\"term\":{\"dataObject.Bill.billDetails.billAccountDetails.taxHeadCode.keyword\":\"TL_TAX\"}},\"aggs\":{\"amount\":{\"sum\":{\"field\":\"dataObject.Bill.billDetails.billAccountDetails.amount\"}}}},\"Adhoc Penalty\":{\"filter\":{\"term\":{\"dataObject.Bill.billDetails.billAccountDetails.taxHeadCode.keyword\":\"TL_ADHOC_PENALTY\"}},\"aggs\":{\"amount\":{\"sum\":{\"field\":\"dataObject.Bill.billDetails.billAccountDetails.amount\"}}}},\"Adhoc Rebate\":{\"filter\":{\"term\":{\"dataObject.Bill.billDetails.billAccountDetails.taxHeadCode.keyword\":\"TL_ADHOC_REBATE\"}},\"aggs\":{\"amount\":{\"sum\":{\"field\":\"dataObject.Bill.billDetails.billAccountDetails.amount\"}}}}}}}}}}" + } + ], + "filterKeys": [ + {"key": "tenantId", "column": "Boundary"} + ], + "chartType": "table", + "valueType": "number", + "drillChart": "licenceTaxHeadsBreakupWard", + "documentType": "_doc", + "action": "", + "plotLabel": "Boundary", + "aggregationPaths": [ + "Tax", + "Adhoc Penalty", + "Adhoc Rebate" + ], + "pathDataTypeMapping": [ + { + "Tax": "amount" + }, + { + "Adhoc Penalty": "amount" + }, + { + "Adhoc Rebate": "amount" + } + ], + "insight": { + }, + "_comment": "" + }, + "licenceTaxHeadsBreakupWard": { + "chartName": "DSS_TL_TAX_HEAD_BREAKUP_BOUNDARY", + "queries": [ + { + "module": "COMMON", + "requestQueryMap": "{\"module\" : \"dataObject.Bill.billDetails.businessService.keyword\", \"tenantId\" : \"dataObject.tenantId\", \"district\" : \"dataObject.tenantData.cityDistrictCode\" }", + "dateRefField": "dataObject.Bill.billDetails.receiptDate", + "indexName": "dss-collection_v1", + "aggrQuery": "{\"aggs\":{\"AGGR\":{\"filter\":{\"bool\":{\"must_not\":[{\"term\":{\"dataObject.tenantId.keyword\":\"pb.testing\"}}],\"must\":[{\"term\":{\"dataObject.Bill.billDetails.status.keyword\":\"Approved\"}}]}},\"aggs\":{\"ULBs \":{\"terms\":{\"field\":\"dataObject.ward.name.keyword\",\"size\":200},\"aggs\":{\"Tax\":{\"filter\":{\"term\":{\"dataObject.Bill.billDetails.billAccountDetails.taxHeadCode.keyword\":\"TL_TAX\"}},\"aggs\":{\"amount\":{\"sum\":{\"field\":\"dataObject.Bill.billDetails.billAccountDetails.amount\"}}}},\"Adhoc Penalty\":{\"filter\":{\"term\":{\"dataObject.Bill.billDetails.billAccountDetails.taxHeadCode.keyword\":\"TL_ADHOC_PENALTY\"}},\"aggs\":{\"amount\":{\"sum\":{\"field\":\"dataObject.Bill.billDetails.billAccountDetails.amount\"}}}},\"Adhoc Rebate\":{\"filter\":{\"term\":{\"dataObject.Bill.billDetails.billAccountDetails.taxHeadCode.keyword\":\"TL_ADHOC_REBATE\"}},\"aggs\":{\"amount\":{\"sum\":{\"field\":\"dataObject.Bill.billDetails.billAccountDetails.amount\"}}}}}}}}}}" + } + ], + "filterKeys": [ + ], + "chartType": "table", + "valueType": "number", + "drillChart": "none", + "documentType": "_doc", + "action": "", + "plotLabel": "Boundary", + "aggregationPaths": [ + "Tax", + "Adhoc Penalty", + "Adhoc Rebate" + ], + "pathDataTypeMapping": [ + { + "Tax": "amount" + }, + { + "Adhoc Penalty": "amount" + }, + { + "Adhoc Rebate": "amount" + } + ], + "insight": { + }, + "_comment": "" + }, + + "_comment": "TL NON-COLLECTION CHARTS", + "licenseIssued": { + "chartName": "DSS_TL_LICENSE_ISSUED", + "queries": [ + { + "module": "TL", + "indexName": "tlindex-v1", + "aggrQuery": "{\"aggs\":{\"AGGR\":{\"filter\":{\"bool\":{\"must_not\":[{\"term\":{\"Data.tradelicense.tenantId.keyword\":\"pb.testing\"}}]}},\"aggs\": {\"License Issued\": {\"value_count\": {\"field\": \"Data.tradelicense.licensenumber.keyword\"}}}}}}", + "requestQueryMap": "{\r\n \"district\" : \"Data.tenantData.city.districtCode\", \r\n\"tenantId\" : \"Data.tradelicense.tenantid\" \r\n, \r\n\"businessService\" : \"dataObject.Bill.billDetails.businessServices\" \r\n}", + "dateRefField": "Data.tradelicense.applicationdate" + } + ], + "chartType": "metric", + "valueType": "number", + "action": "", + "documentType": "_doc", + "drillChart": "none", + "aggregationPaths": [ + "License Issued" + ], + "insight": { + "chartResponseMap" : "licenseIssued", + "action" : "differenceOfNumbers", + "upwardIndicator" : "positive", + "downwardIndicator" : "negative", + "textMessage" : "$indicator$value% than last $insightInterval", + "colorCode" : "#228B22", + "insightInterval" : "month" + }, + "_comment": " licenseIssued is the Chart ID" + }, + "cumulativeLicenseIssued": { + "chartName": "DSS_TL_CUMULATIVE_LICENSE_ISSUED", + "queries": [ + { + "module": "TL", + "indexName": "tlindex-v1", + "aggrQuery": "{\"aggs\":{\"AGGR\":{\"filter\":{\"bool\":{\"must_not\":[{\"term\":{\"Data.tradelicense.tenantId.keyword\":\"pb.testing\"}}]}},\"aggs\":{\"Trade Licence Issued\":{\"date_histogram\":{\"field\":\"Data.tradelicense.issueddate\",\"interval\":\"intervalvalue\"},\"aggs\":{\"Count\":{\"value_count\":{\"field\":\"Data.tradelicense.licensenumber.keyword\"}}}}}}}}", + "requestQueryMap": "{\"tenantId\" : \"Data.tenantData.code\" ,\"district\" : \"Data.tenantData.city.districtCode\"}", + "dateRefField": "Data.tradelicense.issueddate" + }, + { + "module": "TL", + "indexName": "tlindex-v1", + "aggrQuery": "{\"aggs\":{\"AGGR\":{\"filter\":{\"bool\":{\"must_not\":[{\"term\":{\"Data.tradelicense.tenantId.keyword\":\"pb.testing\"}}]}},\"aggs\":{\"Total Application\":{\"date_histogram\":{\"field\":\"Data.tradelicense.applicationdate\",\"interval\":\"intervalvalue\"},\"aggs\":{\"Count\":{\"value_count\":{\"field\":\"Data.tradelicense.applicationnumber.keyword\"}}}}}}}}", + "requestQueryMap": "{\"tenantId\" : \"Data.tenantData.code\" ,\"district\" : \"Data.tenantData.city.districtCode\"}", + "dateRefField": "Data.tradelicense.applicationdate" + } + ], + "chartType": "line", + "valueType": "number", + "action": "", + "documentType": "_doc", + "drillChart": "none", + "aggregationPaths": [ + "Trade Licence Issued", + "Total Application" + ], + "isCumulative": true, + "interval": "month", + "insight": { + }, + "_comment": " Total Number of License having issued" + }, + "licenseApplicationByStatus": { + "chartName": "DSS_TL_LICENSE_BY_STATUS", + "queries": [ + { + "module": "TL", + "requestQueryMap": "{\"module\" : \"dataObject.Bill.billDetails.businessService.keyword\", \"tenantId\" : \"Data.tenantData.code\", \"district\" : \"Data.tenantData.cityDistrictCode\"}", + "dateRefField": "Data.tradelicense.issueddate", + "indexName": "tlindex-v1", + "aggrQuery": "{\"aggs\":{\"AGGR\":{\"filter\":{\"bool\":{\"must_not\":[{\"term\":{\"Data.tradelicense.tenantId.keyword\":\"pb.testing\"}}]}},\"aggs\":{\"Status\":{\"terms\":{\"field\":\"Data.tradelicense.status.keyword\"},\"aggs\":{\"License Issued\":{\"value_count\":{\"field\":\"Data.tradelicense.status.keyword\"}}}}}}}}" + } + ], + "chartType": "pie", + "valueType": "number", + "action": "", + "documentType": "_doc", + "drillChart": "none", + "aggregationPaths": [ + "Status" + ], + "insight": { + }, + "_comment": " License Application which are issued by License Type" + }, + "slaAchieved": { + "chartName": "DSS_SLA_ACHIEVED", + "queries": [ + { + "module": "PGR", + "requestQueryMap": "{\r\n \"module\" : \"businessService.keyword\", \n\"tenantId\" : \"Data.tenantId\"}", + "dateRefField": "Data.dateOfComplaint", + "indexName": "pgrindex-v1", + "aggrQuery": "{\"aggs\":{\"AGGR\":{\"filter\":{\"bool\":{\"must_not\":[{\"term\":{\"Data.tenantId.keyword\":\"pb.testing\"}}]}},\"aggs\":{\"Total Application\":{\"value_count\":{\"field\":\"Data.tenantId.keyword\"}}, \"withinSLA\":{\"range\":{\"script\":{\"lang\":\"painless\",\"source\":\"doc['Data.slaHours'].value\"},\"ranges\":[{\"key\":\"Within SLA\",\"from\":0,\"to\":360}]}}}}}}" + } + ], + "chartType": "metric", + "valueType": "percentage", + "drillChart": "none", + "documentType": "_doc", + "action": "percentage", + "aggregationPaths": [ + "Total Application", + "withinSLA" + ], + "insight": { + "chartResponseMap" : "slaAchieved", + "action" : "differenceOfNumbers", + "upwardIndicator" : "positive", + "downwardIndicator" : "negative", + "textMessage" : "$indicator$value% than last $insightInterval", + "colorCode" : "#228B22", + "insightInterval" : "month" + }, + "_comment": " SLA Achieved Percentage " + }, + "tlStatusByDDR": { + "chartName": "DSS_TL_STATUS_BY_DDR", + "queries": [ + { + "module": "TL", + "requestQueryMap": "{\"tenantId\" : \"Data.tradelicense.tenantId.keyword\" }", + "dateRefField": "Data.tradelicense.applicationDate", + "indexName": "tlindex-v1", + "aggrQuery": "{\"aggs\":{\"AGGR\":{\"filter\":{\"bool\":{\"must_not\":[{\"term\":{\"Data.tradelicense.tenantId.keyword\":\"pb.testing\"}}]}},\"aggs\":{\"Status\":{\"filters\":{\"filters\":{\"Approved\":{\"match\":{\"Data.tradelicense.status.keyword\":\"APPROVED\"}},\"Initiated\":{\"match\":{\"Data.tradelicense.status.keyword\":\"INITIATED\"}},\"Applied\":{\"match\":{\"Data.tradelicense.status.keyword\":\"APPLIED\"}},\"Rejected\":{\"match\":{\"Data.tradelicense.status.keyword\":\"REJECTED\"}},\"Pending Approval\":{\"match\":{\"Data.tradelicense.status.keyword\":\"PENDINGAPPROVAL\"}},\"Field Inspection\":{\"match\":{\"Data.tradelicense.status.keyword\":\"FIELDINSPECTION\"}},\"Cancelled\":{\"match\":{\"Data.tradelicense.status.keyword\":\"CANCELLED\"}},\"Pending Payment\":{\"match\":{\"Data.tradelicense.status.keyword\":\"PENDINGPAYMENT\"}},\"Paid\":{\"match\":{\"Data.tradelicense.status.keyword\":\"PAID\"}}}}}}}}}" + } + ], + "isMdmsEnabled": true, + "filterKeys": [ + {"key": "tenantId", "column": "DDRs"} + ], + "chartType": "table", + "valueType": "number", + "drillChart": "tlStatusByTenant", + "drillFields": [ + "" + ], + "documentType": "_doc", + "action": "", + "plotLabel": "DDRs", + "aggregationPaths": [ + "Approved", + "Initiated", + "Applied", + "Rejected", + "Pending Approval", + "Field Inspection", + "Cancelled", + "Pending Payment", + "Paid" + ], + "pathDataTypeMapping": [ + { + "Approved": "number" + }, + { + "Initiated": "number" + }, + { + "Applied": "number" + }, + { + "Rejected": "number" + }, + { + "Pending Approval": "number" + }, + { + "Field Inspection": "number" + }, + { + "Cancelled": "number" + }, + { + "Pending Payment": "number" + }, + { + "Paid": "number" + } + ], + "insight": { + }, + "_comment": "" + }, + "tlStatusByTenant": { + "chartName": "DSS_TL_STATUS_BY_TENANT", + "queries": [ + { + "module": "TL", + "requestQueryMap": "{\"tenantId\" : \"Data.tradelicense.tenantId.keyword\" }", + "dateRefField": "Data.tradelicense.applicationDate", + "indexName": "tlindex-v1", + "aggrQuery": "{\"aggs\":{\"AGGR\":{\"filter\":{\"bool\":{\"must_not\":[{\"term\":{\"Data.tradelicense.tenantId.keyword\":\"pb.testing\"}}]}},\"aggs\":{\"ULBs \":{\"terms\":{\"field\":\"Data.tradelicense.tenantId.keyword\",\"size\":1000},\"aggs\":{\"Status\":{\"filters\":{\"filters\":{\"Approved\":{\"match\":{\"Data.tradelicense.status.keyword\":\"APPROVED\"}},\"Initiated\":{\"match\":{\"Data.tradelicense.status.keyword\":\"INITIATED\"}},\"Applied\":{\"match\":{\"Data.tradelicense.status.keyword\":\"APPLIED\"}},\"Rejected\":{\"match\":{\"Data.tradelicense.status.keyword\":\"REJECTED\"}},\"Pending Approval\":{\"match\":{\"Data.tradelicense.status.keyword\":\"PENDINGAPPROVAL\"}},\"Field Inspection\":{\"match\":{\"Data.tradelicense.status.keyword\":\"FIELDINSPECTION\"}},\"Cancelled\":{\"match\":{\"Data.tradelicense.status.keyword\":\"CANCELLED\"}},\"Pending Payment\":{\"match\":{\"Data.tradelicense.status.keyword\":\"PENDINGPAYMENT\"}},\"Paid\":{\"match\":{\"Data.tradelicense.status.keyword\":\"PAID\"}}}}}}}}}}}" + } + ], + "filterKeys": [ + {"key": "tenantId", "column": "Boundary"} + ], + "chartType": "table", + "valueType": "number", + "drillChart": "tlStatusByTenantDrillDown", + "drillFields": [ + "" + ], + "documentType": "_doc", + "action": "", + "plotLabel": "ULBs", + "aggregationPaths": [ + "Approved", + "Initiated", + "Applied", + "Rejected", + "Pending Approval", + "Field Inspection", + "Cancelled", + "Pending Payment", + "Paid" + ], + "pathDataTypeMapping": [ + { + "Approved": "number" + }, + { + "Initiated": "number" + }, + { + "Applied": "number" + }, + { + "Rejected": "number" + }, + { + "Pending Approval": "number" + }, + { + "Field Inspection": "number" + }, + { + "Cancelled": "number" + }, + { + "Pending Payment": "number" + }, + { + "Paid": "number" + } + ], + "insight": { + }, + "_comment": "" + }, + "tlStatusByTenantDrillDown": { + "chartName": "DSS_TL_STATUS_BY_WARD", + "queries": [ + { + "module": "TL", + "requestQueryMap": "{\"tenantId\" : \"Data.tradelicense.tenantId.keyword\" }", + "dateRefField": "Data.tradelicense.applicationDate", + "indexName": "tlindex-v1", + "aggrQuery": "{\"aggs\":{\"AGGR\":{\"filter\":{\"bool\":{\"must_not\":[{\"term\":{\"Data.tradelicense.tenantId.keyword\":\"pb.testing\"}}]}},\"aggs\":{\"Ward \":{\"terms\":{\"field\":\"Data.ward.name.keyword\",\"size\":1000},\"aggs\":{\"Status\":{\"filters\":{\"filters\":{\"Approved\":{\"match\":{\"Data.tradelicense.status.keyword\":\"APPROVED\"}},\"Initiated\":{\"match\":{\"Data.tradelicense.status.keyword\":\"INITIATED\"}},\"Applied\":{\"match\":{\"Data.tradelicense.status.keyword\":\"APPLIED\"}},\"Rejected\":{\"match\":{\"Data.tradelicense.status.keyword\":\"REJECTED\"}},\"Pending Approval\":{\"match\":{\"Data.tradelicense.status.keyword\":\"PENDINGAPPROVAL\"}},\"Field Inspection\":{\"match\":{\"Data.tradelicense.status.keyword\":\"FIELDINSPECTION\"}},\"Cancelled\":{\"match\":{\"Data.tradelicense.status.keyword\":\"CANCELLED\"}},\"Pending Payment\":{\"match\":{\"Data.tradelicense.status.keyword\":\"PENDINGPAYMENT\"}},\"Paid\":{\"match\":{\"Data.tradelicense.status.keyword\":\"PAID\"}}}}}}}}}}}" + } + ], + "filterKeys": [ + ], + "chartType": "table", + "valueType": "number", + "drillChart": "", + "drillFields": [ + "" + ], + "documentType": "_doc", + "action": "", + "plotLabel": "Ward", + "aggregationPaths": [ + "Approved", + "Initiated", + "Applied", + "Rejected", + "Pending Approval", + "Field Inspection", + "Cancelled", + "Pending Payment", + "Paid" + ], + "pathDataTypeMapping": [ + { + "Approved": "number" + }, + { + "Initiated": "number" + }, + { + "Applied": "number" + }, + { + "Rejected": "number" + }, + { + "Pending Approval": "number" + }, + { + "Field Inspection": "number" + }, + { + "Cancelled": "number" + }, + { + "Pending Payment": "number" + }, + { + "Paid": "number" + } + ], + "insight": { + }, + "_comment": "" + }, + + "_comment": "Enhanced table charts..TODO: refactor to remove xtable type ", + "xlicenseIssuedBoundaryRevenue": { + "chartName": "DSS_TL_DEMAND_COLLECTION_BOUNDARY", + "queries": [ + { + "module": "COMMON", + "requestQueryMap": "{\"module\" : \"dataObject.Bill.billDetails.businessService.keyword\", \"tenantId\" : \"dataObject.tenantId\", \"district\" : \"dataObject.tenantData.cityDistrictCode\" }", + "dateRefField": "dataObject.Bill.billDetails.receiptDate", + "indexName": "dss-collection_v1", + "aggrQuery": "{\"aggs\":{\"AGGR\":{\"filter\":{\"bool\":{\"must_not\":[{\"term\":{\"dataObject.tenantId.keyword\":\"pb.testing\"}}],\"must\":[{\"term\":{\"dataObject.Bill.billDetails.status.keyword\":\"Approved\"}}]}},\"aggs\":{\"ULBs \":{\"terms\":{\"field\":\"dataObject.tenantId.keyword\",\"size\":200,\"min_doc_count\":0},\"aggs\":{\"Total Collection\":{\"sum\":{\"field\":\"dataObject.Bill.billDetails.amountPaid\"}},\"Transactions\":{\"value_count\":{\"field\":\"dataObject.transactionId.keyword\"}},\"Total Licence Issued\":{\"value_count\":{\"field\":\"domainObject.tradelicense.licensenumber.keyword\"}}}}}}}}" + }, + { + "module": "COMMON", + "requestQueryMap": "{\"module\" : \"businessService.keyword\", \"tenantId\" : \"tenantIdForMunicipalCorporation.keyword\"}", + "dateRefField": "", + "indexName": "dss-target_v1", + "aggrQuery": "{\"aggs\":{\"ULBs \":{\"terms\":{\"field\":\"tenantIdForMunicipalCorporation.keyword\",\"size\":1000,\"min_doc_count\":0},\"aggs\":{\"Target Collection\":{\"sum\":{\"field\":\"budgetProposedForMunicipalCorporation\"}}}}}}" + } + ], + "filterKeys": [ + {"key": "tenantId", "column": "Boundary"} + ], + "postAggregationTheory" : "repsonseToDifferenceOfDates", + "chartType": "xtable", + "valueType": "number", + "drillChart": "", + "plotLabel": "Boundary", + "aggregationPaths": [ + ], + "computedFields": [ + { + "postAggregationTheory" : "repsonseToDifferenceOfDates", + "actionName": "PercentageComputedField", + "fields" : ["Target Collection", "Total Collection"], + "newField" : "Target Achieved", + "_comments": "fields are field names picked from its aggregation query to use post aggregation newField value with given new field name " + }, + { + "postAggregationTheory" : "repsonseToDifferenceOfDates", + "actionName": "PercentageComputedField", + "fields" : ["Target Collection", "Total Collection"], + "newField" : "Target Achieved", + "_comments": "fields are field names picked from its aggregation query to use post aggregation newField value with given new field name " + } + ], + "pathDataTypeMapping": [ + ], + "insight": { + }, + "_comment": "" + }, + "xptFyByDDR": { + "chartName": "DSS_TL_STATUS_BY_TENANT", + "queries": [ + { + "module": "PT", + "requestQueryMap": "{\"tenantId\" : \"Data.tenantId.keyword\" }", + "dateRefField": "Data.@timestamp", + "indexName": "ptindex-v1", + "aggrQuery": "{\"aggs\":{\"AGGR\":{\"filter\":{\"bool\":{\"must_not\":[{\"term\":{\"Data.tenantId.keyword\":\"pb.testing\"}}]}},\"aggs\":{\"Fys\":{\"terms\":{\"field\":\"Data.propertyDetails.financialYear.keyword\",\"min_doc_count\":0}}}}}}" + }, + { + "module": "PT", + "requestQueryMap": "{\"tenantId\" : \"Data.tenantId.keyword\" }", + "dateRefField": "Data.@timestamp", + "indexName": "ptindex-v1", + "aggrQuery": "{\"aggs\":{\"AGGR\":{\"filter\":{\"bool\":{\"must_not\":[{\"term\":{\"Data.tenantId.keyword\":\"pb.testing\"}}]}},\"aggs\":{\"Total\":{\"value_count\":{\"field\":\"Data.propertyDetails.assessmentNumber.keyword\"}}}}}}}}" + } + ], + "isMdmsEnabled": true, + "filterKeys": [ + {"key": "tenantId", "column": "DDRs"} + ], + "chartType": "xtable", + "valueType": "number", + "drillChart": "xptFyByTenant", + "plotLabel": "DDRs", + "insight": { + }, + "_comment": "" + }, + "xptFyByTenant": { + "chartName": "DSS_TL_STATUS_BY_TENANT", + "queries": [ + { + "module": "PT", + "requestQueryMap": "{\"tenantId\" : \"Data.tenantId.keyword\" }", + "dateRefField": "Data.@timestamp", + "indexName": "ptindex-v1", + "aggrQuery": "{\"aggs\":{\"AGGR\":{\"filter\":{\"bool\":{\"must_not\":[{\"term\":{\"Data.tenantId.keyword\":\"pb.testing\"}}]}},\"aggs\":{\"ULBS \":{\"terms\":{\"field\":\"Data.tenantId.keyword\"},\"aggs\":{\"Fys\":{\"terms\":{\"field\":\"Data.propertyDetails.financialYear.keyword\",\"min_doc_count\":0}}}}}}}}" + }, + { + "module": "PT", + "requestQueryMap": "{\"tenantId\" : \"Data.tenantId.keyword\" }", + "dateRefField": "Data.@timestamp", + "indexName": "ptindex-v1", + "aggrQuery": "{\"aggs\":{\"AGGR\":{\"filter\":{\"bool\":{\"must_not\":[{\"term\":{\"Data.tenantId.keyword\":\"pb.testing\"}}]}},\"aggs\":{\"ULBS \":{\"terms\":{\"field\":\"Data.tenantId.keyword\"},\"aggs\":{\"Total\":{\"value_count\":{\"field\":\"Data.propertyDetails.assessmentNumber.keyword\"}}}}}}}}" + } + ], + "filterKeys": [ + {"key": "tenantId", "column": "Boundary"} + ], + "chartType": "xtable", + "valueType": "number", + "drillChart": "xptFyByWard", + "plotLabel": "Boundary", + "insight": { + }, + "_comment": "" + }, + "xptFyByWard": { + "chartName": "DSS_TL_STATUS_BY_TENANT", + "queries": [ + { + "module": "PT", + "requestQueryMap": "{\"tenantId\" : \"Data.tenantId.keyword\" }", + "dateRefField": "Data.@timestamp", + "indexName": "ptindex-v1", + "aggrQuery": "{\"aggs\":{\"AGGR\":{\"filter\":{\"bool\":{\"must_not\":[{\"term\":{\"Data.tenantId.keyword\":\"pb.testing\"}}]}},\"aggs\":{\"Ward \":{\"terms\":{\"field\":\"Data.ward.name.keyword\"},\"aggs\":{\"Fys\":{\"terms\":{\"field\":\"Data.propertyDetails.financialYear.keyword\",\"min_doc_count\":0}}}}}}}}" + }, + { + "module": "PT", + "requestQueryMap": "{\"tenantId\" : \"Data.tenantId.keyword\" }", + "dateRefField": "Data.@timestamp", + "indexName": "ptindex-v1", + "aggrQuery": "{\"aggs\":{\"AGGR\":{\"filter\":{\"bool\":{\"must_not\":[{\"term\":{\"Data.tenantId.keyword\":\"pb.testing\"}}]}},\"aggs\":{\"Ward \":{\"terms\":{\"field\":\"Data.ward.name.keyword\"},\"aggs\":{\"Total\":{\"value_count\":{\"field\":\"Data.propertyDetails.assessmentNumber.keyword\"}}}}}}}}" + } + ], + "filterKeys": [ + ], + "chartType": "xtable", + "valueType": "number", + "drillChart": "none", + "plotLabel": "Wards", + "insight": { + }, + "_comment": "" + }, + + "xpgrStatusByDDR": { + "chartName": "DSS_PGR_STATUS_BY_TENANT", + "queries": [ + { + "module": "PGR", + "requestQueryMap": "{\"tenantId\" : \"Data.tenantId.keyword\" , \"departmentId\" : \"Data.department.keyword\"}", + "dateRefField": "Data.dateOfComplaint", + "indexName": "pgrindex-v1", + "aggrQuery": "{\"aggs\":{\"AGGR\":{\"filter\":{\"bool\":{\"must_not\":[{\"term\":{\"Data.tenantId.keyword\":\"pb.testing\"}}]}},\"aggs\":{\"withinSLA\":{\"range\":{\"script\":{\"lang\":\"painless\",\"source\":\"doc['Data.slaHours'].value\"},\"ranges\":[{\"key\":\"withinSLA\",\"from\":0,\"to\":360}]}},\"Total Complaints\":{\"value_count\":{\"field\":\"Data.dateOfComplaint\"}},\"Closed_Complaints\":{\"filter\":{\"terms\":{\"Data.status.keyword\":[\"closed\",\"rejected\",\"resolved\"]}},\"aggs\":{\"Closed Complaints\":{\"value_count\":{\"field\":\"Data.tenantId.keyword\"}}}},\"Open_Complaints\":{\"filter\":{\"terms\":{\"Data.status.keyword\":[\"open\"]}},\"aggs\":{\"Open Complaints\":{\"value_count\":{\"field\":\"Data.tenantId.keyword\"}}}},\"Reopened_Complaints\":{\"filter\":{\"terms\":{\"Data.status.keyword\":[\"reopen\"]}},\"aggs\":{\"Reopened Complaints\":{\"value_count\":{\"field\":\"Data.tenantId.keyword\"}}}}}}}}" + } + ], + "isMdmsEnabled": true, + "filterKeys": [ + {"key": "tenantId", "column": "DDRs"} + ], + "chartType": "xtable", + "valueType": "number", + "drillChart": "xpgrStatusByTenant", + "plotLabel": "DDRs", + "excludedColumns": ["withinSLA"], + "computedFields": [ + { + "postAggregationTheory" : "", + "actionName": "PercentageComputedField", + "fields" : ["Closed Complaints", "Total Complaints"], + "newField" : "Completion Rate", + "_comments": "fields are field names picked from its aggregation query to use post aggregation newField value with given new field name " + }, + { + "postAggregationTheory" : "", + "actionName": "PercentageComputedField", + "fields" : ["Total Complaints", "withinSLA"], + "newField" : "Sla Achieved", + "_comments": "fields are field names picked from its aggregation query to use post aggregation newField value with given new field name " + } + ], + "insight": { + }, + "_comment": "" + }, + "xpgrStatusByTenant": { + "chartName": "DSS_PGR_STATUS_BY_TENANT", + "queries": [ + { + "module": "PGR", + "requestQueryMap": "{\"tenantId\" : \"Data.tenantId.keyword\" , \"departmentId\" : \"Data.department.keyword\"}", + "dateRefField": "Data.dateOfComplaint", + "indexName": "pgrindex-v1", + "aggrQuery": "{\"aggs\":{\"AGGR\":{\"filter\":{\"bool\":{\"must_not\":[{\"term\":{\"Data.tenantId.keyword\":\"pb.testing\"}}]}},\"aggs\":{\"ULBs \":{\"terms\":{\"field\":\"Data.tenantId.keyword\",\"size\":1000},\"aggs\":{\"withinSLA\":{\"range\":{\"script\":{\"lang\":\"painless\",\"source\":\"doc['Data.slaHours'].value\"},\"ranges\":[{\"key\":\"withinSLA\",\"from\":0,\"to\":360}]}},\"Total Complaints\":{\"value_count\":{\"field\":\"Data.dateOfComplaint\"}},\"Closed_Complaints\":{\"filter\":{\"terms\":{\"Data.status.keyword\":[\"closed\",\"rejected\",\"resolved\"]}},\"aggs\":{\"Closed Complaints\":{\"value_count\":{\"field\":\"Data.tenantId.keyword\"}}}},\"Open_Complaints\":{\"filter\":{\"terms\":{\"Data.status.keyword\":[\"open\"]}},\"aggs\":{\"Open Complaints\":{\"value_count\":{\"field\":\"Data.tenantId.keyword\"}}}},\"Reopened_Complaints\":{\"filter\":{\"terms\":{\"Data.status.keyword\":[\"reopen\"]}},\"aggs\":{\"Reopened Complaints\":{\"value_count\":{\"field\":\"Data.tenantId.keyword\"}}}}}}}}}}" + } + ], + "filterKeys": [ + {"key": "tenantId", "column": "Boundary"} + ], + "chartType": "xtable", + "valueType": "number", + "drillChart": "xpgrStatusByWard", + "plotLabel": "Boundary", + "excludedColumns": ["withinSLA"], + "computedFields": [ + { + "postAggregationTheory" : "", + "actionName": "PercentageComputedField", + "fields" : ["Closed Complaints", "Total Complaints"], + "newField" : "Completion Rate", + "_comments": "fields are field names picked from its aggregation query to use post aggregation newField value with given new field name " + }, + { + "postAggregationTheory" : "", + "actionName": "PercentageComputedField", + "fields" : ["Total Complaints", "withinSLA"], + "newField" : "Sla Achieved", + "_comments": "fields are field names picked from its aggregation query to use post aggregation newField value with given new field name " + } + ], + "insight": { + }, + "_comment": "" + }, + "xpgrStatusByWard": { + "chartName": "DSS_PGR_STATUS_BY_TENANT", + "queries": [ + { + "module": "PGR", + "requestQueryMap": "{\"tenantId\" : \"Data.tenantId.keyword\" , \"departmentId\" : \"Data.department.keyword\"}", + "dateRefField": "Data.dateOfComplaint", + "indexName": "pgrindex-v1", + "aggrQuery": "{\"aggs\":{\"AGGR\":{\"filter\":{\"bool\":{\"must_not\":[{\"term\":{\"Data.tenantId.keyword\":\"pb.testing\"}}]}},\"aggs\":{\"Ward \":{\"terms\":{\"field\":\"Data.complaintWard.name.keyword\",\"size\":1000},\"aggs\":{\"withinSLA\":{\"range\":{\"script\":{\"lang\":\"painless\",\"source\":\"doc['Data.slaHours'].value\"},\"ranges\":[{\"key\":\"withinSLA\",\"from\":0,\"to\":360}]}},\"Total Complaints\":{\"value_count\":{\"field\":\"Data.dateOfComplaint\"}},\"Closed_Complaints\":{\"filter\":{\"terms\":{\"Data.status.keyword\":[\"closed\",\"rejected\",\"resolved\"]}},\"aggs\":{\"Closed Complaints\":{\"value_count\":{\"field\":\"Data.tenantId.keyword\"}}}},\"Open_Complaints\":{\"filter\":{\"terms\":{\"Data.status.keyword\":[\"open\"]}},\"aggs\":{\"Open Complaints\":{\"value_count\":{\"field\":\"Data.tenantId.keyword\"}}}},\"Reopened_Complaints\":{\"filter\":{\"terms\":{\"Data.status.keyword\":[\"reopen\"]}},\"aggs\":{\"Reopened Complaints\":{\"value_count\":{\"field\":\"Data.tenantId.keyword\"}}}}}}}}}}" + } + ], + "filterKeys": [ + {"key": "tenantId", "column": "Boundary"} + ], + "chartType": "xtable", + "valueType": "number", + "drillChart": "none", + + "documentType": "_doc", + "action": "", + "plotLabel": "Ward", + "excludedColumns": ["withinSLA"], + "computedFields": [ + { + "postAggregationTheory" : "", + "actionName": "PercentageComputedField", + "fields" : ["Closed Complaints", "Total Complaints"], + "newField" : "Completion Rate", + "_comments": "fields are field names picked from its aggregation query to use post aggregation newField value with given new field name " + }, + { + "postAggregationTheory" : "", + "actionName": "PercentageComputedField", + "fields" : ["Total Complaints", "withinSLA"], + "newField" : "Sla Achieved", + "_comments": "fields are field names picked from its aggregation query to use post aggregation newField value with given new field name " + } + ], + "insight": { + }, + "_comment": "" + }, + "xpgrStatusByDept": { + "chartName": "DSS_PGR_STATUS_BY_TENANT", + "queries": [ + { + "module": "PGR", + "requestQueryMap": "{\"tenantId\" : \"Data.tenantId.keyword\" , \"departmentId\" : \"Data.department.keyword\"}", + "dateRefField": "Data.dateOfComplaint", + "indexName": "pgrindex-v1", + "aggrQuery": "{\"aggs\":{\"AGGR\":{\"filter\":{\"bool\":{\"must_not\":[{\"term\":{\"Data.tenantId.keyword\":\"pb.testing\"}}]}},\"aggs\":{\"Department \":{\"terms\":{\"field\":\"Data.department.keyword\",\"size\":1000},\"aggs\":{\"withinSLA\":{\"range\":{\"script\":{\"lang\":\"painless\",\"source\":\"doc['Data.slaHours'].value\"},\"ranges\":[{\"key\":\"withinSLA\",\"from\":0,\"to\":360}]}},\"Total Complaints\":{\"value_count\":{\"field\":\"Data.dateOfComplaint\"}},\"Closed_Complaints\":{\"filter\":{\"terms\":{\"Data.status.keyword\":[\"closed\",\"rejected\",\"resolved\"]}},\"aggs\":{\"Closed Complaints\":{\"value_count\":{\"field\":\"Data.tenantId.keyword\"}}}},\"Open_Complaints\":{\"filter\":{\"terms\":{\"Data.status.keyword\":[\"open\"]}},\"aggs\":{\"Open Complaints\":{\"value_count\":{\"field\":\"Data.tenantId.keyword\"}}}},\"Reopened_Complaints\":{\"filter\":{\"terms\":{\"Data.status.keyword\":[\"reopen\"]}},\"aggs\":{\"Reopened Complaints\":{\"value_count\":{\"field\":\"Data.tenantId.keyword\"}}}}}}}}}}" + } + ], + "filterKeys": [ + {"key": "departmentId", "column": "Department"} + ], + "chartType": "xtable", + "valueType": "number", + "drillChart": "xpgrStatusByCatagory", + + "documentType": "_doc", + "action": "", + "plotLabel": "Department", + "excludedColumns": ["withinSLA"], + "computedFields": [ + { + "postAggregationTheory" : "", + "actionName": "PercentageComputedField", + "fields" : ["Closed Complaints", "Total Complaints"], + "newField" : "Completion Rate", + "_comments": "fields are field names picked from its aggregation query to use post aggregation newField value with given new field name " + }, + { + "postAggregationTheory" : "", + "actionName": "PercentageComputedField", + "fields" : ["Total Complaints", "withinSLA"], + "newField" : "Sla Achieved", + "_comments": "fields are field names picked from its aggregation query to use post aggregation newField value with given new field name " + } + ], + "insight": { + }, + "_comment": "" + }, + "xpgrStatusByCatagory": { + "chartName": "DSS_PGR_STATUS_BY_TENANT", + "queries": [ + { + "module": "PGR", + "requestQueryMap": "{\"tenantId\" : \"Data.tenantId.keyword\" , \"departmentId\" : \"Data.department.keyword\"}", + "dateRefField": "Data.dateOfComplaint", + "indexName": "pgrindex-v1", + "aggrQuery": "{\"aggs\":{\"AGGR\":{\"filter\":{\"bool\":{\"must_not\":[{\"term\":{\"Data.tenantId.keyword\":\"pb.testing\"}}]}},\"aggs\":{\"Category \":{\"terms\":{\"field\":\"Data.complainCategory.keyword\",\"size\":1000},\"aggs\":{\"withinSLA\":{\"range\":{\"script\":{\"lang\":\"painless\",\"source\":\"doc['Data.slaHours'].value\"},\"ranges\":[{\"key\":\"withinSLA\",\"from\":0,\"to\":360}]}},\"Total Complaints\":{\"value_count\":{\"field\":\"Data.dateOfComplaint\"}},\"Closed_Complaints\":{\"filter\":{\"terms\":{\"Data.status.keyword\":[\"closed\",\"rejected\",\"resolved\"]}},\"aggs\":{\"Closed Complaints\":{\"value_count\":{\"field\":\"Data.tenantId.keyword\"}}}},\"Open_Complaints\":{\"filter\":{\"terms\":{\"Data.status.keyword\":[\"open\"]}},\"aggs\":{\"Open Complaints\":{\"value_count\":{\"field\":\"Data.tenantId.keyword\"}}}},\"Reopened_Complaints\":{\"filter\":{\"terms\":{\"Data.status.keyword\":[\"reopen\"]}},\"aggs\":{\"Reopened Complaints\":{\"value_count\":{\"field\":\"Data.tenantId.keyword\"}}}}}}}}}}" + } + ], + "filterKeys": [ + {"key": "departmentId", "column": "Department"} + ], + "chartType": "xtable", + "valueType": "number", + "drillChart": "none", + + "documentType": "_doc", + "action": "", + "plotLabel": "Complaint Category", + "excludedColumns": ["withinSLA"], + "computedFields": [ + { + "postAggregationTheory" : "", + "actionName": "PercentageComputedField", + "fields" : ["Closed Complaints", "Total Complaints"], + "newField" : "Completion Rate", + "_comments": "fields are field names picked from its aggregation query to use post aggregation newField value with given new field name " + }, + { + "postAggregationTheory" : "", + "actionName": "PercentageComputedField", + "fields" : ["Total Complaints", "withinSLA"], + "newField" : "Sla Achieved", + "_comments": "fields are field names picked from its aggregation query to use post aggregation newField value with given new field name " + } + ], + "insight": { + }, + "_comment": "" + }, + + + + + + "_comment": "PGR NON-COLLECTION CHARTS", + "completionRate": { + "chartName": "DSS_PGR_COMPLETION_RATE", + "queries": [ + { + "module": "PGR", + "requestQueryMap": "{\"tenantId\" : \"Data.tenantId\"}", + "dateRefField": "Data.dateOfComplaint", + "indexName": "pgrindex-v1", + "aggrQuery": "{\"aggs\":{\"AGGR\":{\"filter\":{\"bool\":{\"must_not\":[{\"term\":{\"Data.tenantId.keyword\":\"pb.testing\"}}]}},\"aggs\":{\"Total Complaints\":{\"value_count\":{\"field\":\"Data.dateOfComplaint\"}},\"Closed Complaints\":{\"filters\":{\"filters\":{\"closed\":{\"match\":{\"Data.status.keyword\":\"closed\"}},\"resolved\":{\"match\":{\"Data.status.keyword\":\"resolved\"}},\"rejected\":{\"match\":{\"Data.status.keyword\":\"rejected\"}}}}}}}}}" + } + ], + "chartType": "metric", + "valueType": "percentage", + "drillChart": "none", + "documentType": "_doc", + "action": "percentage", + "aggregationPaths": [ + "Closed Complaints", + "Total Complaints" + ], + "insight": { + "chartResponseMap" : "completionRate", + "action" : "differenceOfNumbers", + "upwardIndicator" : "positive", + "downwardIndicator" : "negative", + "textMessage" : "$indicator$value% than last $insightInterval", + "colorCode" : "#228B22", + "insightInterval" : "month" + }, + "_comment": " Total Number of Complaints " + }, + "totalComplaints": { + "chartName": "DSS_PGR_TOTAL_COMPLAINTS", + "queries": [ + { + "module": "PGR", + "requestQueryMap": "{\"tenantId\" : \"Data.tenantId\"}", + "dateRefField": "Data.dateOfComplaint", + "indexName": "pgrindex-v1", + "aggrQuery": "{\"aggs\":{\"AGGR\":{\"filter\":{\"bool\":{\"must_not\":[{\"term\":{\"Data.tenantId.keyword\":\"pb.testing\"}}]}},\"aggs\":{\"Total Complaints\":{\"value_count\":{\"field\":\"Data.dateOfComplaint\"}}}}}}" + } + ], + "chartType": "metric", + "valueType": "number", + "drillChart": "none", + "documentType": "_doc", + "action": "", + "aggregationPaths": [ + "Total Complaints" + ], + "insight": { + "chartResponseMap" : "totalComplaints", + "action" : "differenceOfNumbers", + "upwardIndicator" : "positive", + "downwardIndicator" : "negative", + "textMessage" : "$indicator$value% than last $insightInterval", + "colorCode" : "#228B22", + "insightInterval" : "month" + }, + "_comment": " Total Number of Complaints " + }, + "closedComplaints": { + "chartName": "DSS_PGR_CLOSED_COMPLAINTS", + "queries": [ + { + "module": "PGR", + "requestQueryMap": "{\"tenantId\" : \"Data.tenantId\"}", + "dateRefField": "Data.dateOfComplaint", + "indexName": "pgrindex-v1", + "aggrQuery": "{\"aggs\":{\"AGGR\":{\"filter\":{\"bool\":{\"must_not\":[{\"term\":{\"Data.tenantId.keyword\":\"pb.testing\"}}]}},\"aggs\":{\"Closed Complaints\":{\"filters\":{\"filters\":{\"closed\":{\"match\":{\"Data.status.keyword\":\"closed\"}},\"resolved\":{\"match\":{\"Data.status.keyword\":\"resolved\"}},\"rejected\":{\"match\":{\"Data.status.keyword\":\"rejected\"}}}}}}}}}" + } + ], + "chartType": "metric", + "valueType": "number", + "drillChart": "none", + "documentType": "_doc", + "action": "", + "aggregationPaths": [ + "Closed Complaints" + ], + "insight": { + "chartResponseMap" : "closedComplaints", + "action" : "differenceOfNumbers", + "upwardIndicator" : "positive", + "downwardIndicator" : "negative", + "textMessage" : "$indicator$value% than last $insightInterval", + "colorCode" : "#228B22", + "insightInterval" : "month" + }, + "_comment": " Total Number of Closed Complaints " + }, + "cumulativeClosedComplaints": { + "chartName": "DSS_TOTAL_CUMULATIVE_CLOSED_COMPLAINTS", + "queries": [ + { + "module": "PGR", + "dateRefField": "Data.dateOfComplaint", + "requestQueryMap": "{\"tenantId\" : \"Data.tenantId\"}", + "indexName": "pgrindex-v1", + "aggrQuery": "{\"aggs\":{\"AGGR\":{\"filter\":{\"bool\":{\"must_not\":[{\"term\":{\"Data.tenantId.keyword\":\"pb.testing\"}}]}},\"aggs\":{\"Total Complaints\":{\"date_histogram\":{\"field\":\"Data.dateOfComplaint\",\"interval\":\"intervalvalue\"},\"aggs\":{\"Count\":{\"value_count\":{\"field\":\"Data.tenantId.keyword\"}}}},\"Closed Complaints\":{\"date_histogram\":{\"field\":\"Data.dateOfComplaint\",\"interval\":\"intervalvalue\"},\"aggs\":{\"Closed\":{\"filter\":{\"terms\":{\"Data.status.keyword\":[\"closed\",\"rejected\",\"resolved\"]}},\"aggs\":{\"Count\":{\"value_count\":{\"field\":\"Data.tenantId.keyword\"}}}}}},\"Reopened Complaints\":{\"date_histogram\":{\"field\":\"Data.dateOfComplaint\",\"interval\":\"intervalvalue\"},\"aggs\":{\"Reopened\":{\"filter\":{\"terms\":{\"Data.status.keyword\":[\"reopened\"]}},\"aggs\":{\"Count\":{\"value_count\":{\"field\":\"Data.tenantId.keyword\"}}}}}}}}}}" + } + ], + "chartType": "line", + "valueType": "number", + "action": "", + "drillChart": "none", + "documentType": "_doc", + "aggregationPaths": [ + "Closed Complaints", + "Reopened Complaints", + "Total Complaints" + + ], + "isCumulative": true, + "interval": "month", + "insight": { + }, + "_comment": " " + }, + "totalComplaintsbySource": { + "chartName": "DSS_PGR_TOTAL_COMPLAINTS_BY_SOURCE", + "queries": [ + { + "module": "PGR", + "dateRefField": "Data.dateOfComplaint", + "requestQueryMap": "{\"tenantId\" : \"Data.tenantId\"}", + "indexName": "pgrindex-v1", + "aggrQuery": "{\"aggs\":{\"AGGR\":{\"filter\":{\"bool\":{\"must_not\":[{\"term\":{\"Data.tenantId.keyword\":\"pb.testing\"}}]}},\"aggs\":{\"Ivr\":{\"date_histogram\":{\"field\":\"Data.dateOfComplaint\",\"interval\":\"intervalvalue\"},\"aggs\":{\"ivr\":{\"filter\":{\"terms\":{\"Data.source.keyword\":[\"ivr\"]}},\"aggs\":{\"Count\":{\"value_count\":{\"field\":\"Data.dateOfComplaint\"}}}}}},\"Web\":{\"date_histogram\":{\"field\":\"Data.dateOfComplaint\",\"interval\":\"intervalvalue\"},\"aggs\":{\"web\":{\"filter\":{\"terms\":{\"Data.source.keyword\":[\"web\"]}},\"aggs\":{\"Count\":{\"value_count\":{\"field\":\"Data.dateOfComplaint\"}}}}}},\"Whatsapp\":{\"date_histogram\":{\"field\":\"Data.dateOfComplaint\",\"interval\":\"intervalvalue\"},\"aggs\":{\"whatapp\":{\"filter\":{\"terms\":{\"Data.source.keyword\":[\"whatsapp\"]}},\"aggs\":{\"Count\":{\"value_count\":{\"field\":\"Data.dateOfComplaint\"}}}}}},\"Mobileapp\":{\"date_histogram\":{\"field\":\"Data.dateOfComplaint\",\"interval\":\"intervalvalue\"},\"aggs\":{\"mobileapp\":{\"filter\":{\"terms\":{\"Data.source.keyword\":[\"mobileapp\"]}},\"aggs\":{\"Count\":{\"value_count\":{\"field\":\"Data.dateOfComplaint\"}}}}}}}}}}" + } + ], + "chartType": "line", + "valueType": "number", + "action": "", + "drillChart": "none", + "documentType": "_doc", + "aggregationPaths": [ + "Ivr", + "Web", + "Whatsapp", + "Mobileapp" + ], + "isCumulative": false, + "interval": "month", + "insight": { + }, + "_comment": " " + }, + "totalComplaintsbyStatus": { + "chartName": "DSS_PGR_TOTAL_COMPLAINTS_BY_STATUS", + "queries": [ + { + "module": "PGR", + "dateRefField": "Data.dateOfComplaint", + "requestQueryMap": "{\"tenantId\" : \"Data.tenantId\"}", + "indexName": "pgrindex-v1", + "aggrQuery": "{\"aggs\":{\"AGGR\":{\"filter\":{\"bool\":{\"must_not\":[{\"term\":{\"Data.tenantId.keyword\":\"pb.testing\"}}]}},\"aggs\":{\"Resolved\":{\"date_histogram\":{\"field\":\"Data.dateOfComplaint\",\"interval\":\"intervalvalue\"},\"aggs\":{\"resolved\":{\"filter\":{\"terms\":{\"Data.status.keyword\":[\"resolved\"]}},\"aggs\":{\"Count\":{\"value_count\":{\"field\":\"Data.dateOfComplaint\"}}}}}},\"Open\":{\"date_histogram\":{\"field\":\"Data.dateOfComplaint\",\"interval\":\"intervalvalue\"},\"aggs\":{\"open\":{\"filter\":{\"terms\":{\"Data.status.keyword\":[\"open\"]}},\"aggs\":{\"Count\":{\"value_count\":{\"field\":\"Data.dateOfComplaint\"}}}}}},\"Closed\":{\"date_histogram\":{\"field\":\"Data.dateOfComplaint\",\"interval\":\"intervalvalue\"},\"aggs\":{\"closed\":{\"filter\":{\"terms\":{\"Data.status.keyword\":[\"closed\"]}},\"aggs\":{\"Count\":{\"value_count\":{\"field\":\"Data.dateOfComplaint\"}}}}}}}}}}" + } + ], + "chartType": "line", + "valueType": "number", + "action": "", + "drillChart": "none", + "documentType": "_doc", + "aggregationPaths": [ + "Resolved", + "Open", + "Closed" + ], + "isCumulative": false, + "interval": "month", + "insight": { + }, + "_comment": " " + }, + "complaintsByStatus": { + "chartName": "DSS_PGR_COMPLAINTS_BY_STATUS", + "queries": [ + { + "module": "PGR", + "requestQueryMap": "{\"tenantId\" : \"Data.tenantId\"}", + "dateRefField": "Data.dateOfComplaint", + "indexName": "pgrindex-v1", + "aggrQuery": "{\"aggs\":{\"AGGR\":{\"filter\":{\"bool\":{\"must_not\":[{\"term\":{\"Data.tenantId.keyword\":\"pb.testing\"}}]}},\"aggs\":{\"Complaints By Status\":{\"terms\":{\"field\":\"Data.status.keyword\"},\"aggs\":{\"Count\":{\"value_count\":{\"field\":\"Data.dateOfComplaint\"}}}}}}}}" + } + ], + "chartType": "pie", + "valueType": "number", + "action": "", + "documentType": "_doc", + "drillChart": "none", + "aggregationPaths": [ + "Complaints By Status" + ], + "insight": { + }, + "_comment": " Complaints By Their Statuses" + }, + "complaintsByChannel": { + "chartName": "DSS_PGR_COMPLAINTS_BY_CHANNELS", + "queries": [ + { + "module": "PGR", + "requestQueryMap": "{\"tenantId\" : \"Data.tenantId\"}", + "dateRefField": "Data.dateOfComplaint", + "indexName": "pgrindex-v1", + "aggrQuery": "{\"aggs\":{\"AGGR\":{\"filter\":{\"bool\":{\"must_not\":[{\"term\":{\"Data.tenantId.keyword\":\"pb.testing\"}}]}},\"aggs\":{\"Complaints By Channels\":{\"terms\":{\"field\":\"Data.source.keyword\"},\"aggs\":{\"Count\":{\"value_count\":{\"field\":\"Data.dateOfComplaint\"}}}}}}}}" + } + ], + "chartType": "pie", + "valueType": "number", + "action": "", + "documentType": "_doc", + "drillChart": "none", + "aggregationPaths": [ + "Complaints By Channels" + ], + "insight": { + }, + "_comment": " Top 3 Channels For Complaints" + }, + "complaintsByCategory": { + "chartName": "DSS_PGR_COMPLAINTS_BY_CATEGORY", + "queries": [ + { + "module": "PGR", + "requestQueryMap": "{\"tenantId\" : \"Data.tenantId\"}", + "dateRefField": "Data.dateOfComplaint", + "indexName": "pgrindex-v1", + "aggrQuery": "{\"aggs\":{\"AGGR\":{\"filter\":{\"bool\":{\"must_not\":[{\"term\":{\"Data.tenantId.keyword\":\"pb.testing\"}}]}},\"aggs\":{\"Complaints By Category\":{\"terms\":{\"field\":\"Data.complainCategory.keyword\"},\"aggs\":{\"Count\":{\"value_count\":{\"field\":\"Data.dateOfComplaint\"}}}}}}}}" + } + ], + "chartType": "pie", + "valueType": "number", + "action": "", + "documentType": "_doc", + "drillChart": "none", + "aggregationPaths": [ + "Complaints By Category" + ], + "insight": { + }, + "_comment": " Top 3 Channels For Complaints" + }, + "complaintsByTenant": { + "chartName": "DSS_PGR_COMPLAINTS_BY_TENANT", + "queries": [ + { + "module": "PGR", + "requestQueryMap": "{\"tenantId\" : \"Data.tenantId\"}", + "dateRefField": "Data.dateOfComplaint", + "indexName": "pgrindex-v1", + "aggrQuery": "{\"aggs\":{\"AGGR\":{\"filter\":{\"bool\":{\"must_not\":[{\"term\":{\"Data.tenantId.keyword\":\"pb.testing\"}}]}},\"aggs\":{\"Complaints By Tenant\":{\"terms\":{\"field\":\"Data.tenantId.keyword\"},\"aggs\":{\"Count\":{\"value_count\":{\"field\":\"Data.dateOfComplaint\"}}}}}}}}" + } + ], + "chartType": "pie", + "valueType": "number", + "action": "", + "documentType": "_doc", + "drillChart": "none", + "aggregationPaths": [ + "Complaints By Tenant" + ], + "insight": { + }, + "_comment": " Complaints by Tenant" + }, + "complaintsByDistrict": { + "chartName": "DSS_PGR_COMPLAINTS_BY_DISTRICT", + "queries": [ + { + "module": "PGR", + "requestQueryMap": "{\"tenantId\" : \"Data.tenantId\"}", + "dateRefField": "Data.dateOfComplaint", + "indexName": "pgrindex-v1", + "aggrQuery": "{\"aggs\":{\"AGGR\":{\"filter\":{\"bool\":{\"must_not\":[{\"term\":{\"Data.tenantId.keyword\":\"pb.testing\"}}]}},\"aggs\":{\"Complaints By District\":{\"terms\":{\"field\":\"Data.tenantData.city.districtName.keyword\"},\"aggs\":{\"Count\":{\"value_count\":{\"field\":\"Data.dateOfComplaint\"}}}}}}}}" + } + ], + "chartType": "pie", + "valueType": "number", + "action": "", + "documentType": "_doc", + "drillChart": "none", + "aggregationPaths": [ + "Complaints By District" + ], + "insight": { + }, + "_comment": " Complaints by District" + }, + "complaintsByDepartment": { + "chartName": "DSS_PGR_COMPLAINTS_BY_DEPARTMENT", + "queries": [ + { + "module": "PGR", + "requestQueryMap": "{\"tenantId\" : \"Data.tenantId\"}", + "dateRefField": "Data.dateOfComplaint", + "indexName": "pgrindex-v1", + "aggrQuery": "{\"aggs\":{\"AGGR\":{\"filter\":{\"bool\":{\"must_not\":[{\"term\":{\"Data.tenantId.keyword\":\"pb.testing\"}}]}},\"aggs\":{\"Complaints By Department\":{\"terms\":{\"field\":\"Data.department.keyword\"},\"aggs\":{\"Count\":{\"value_count\":{\"field\":\"Data.dateOfComplaint\"}}}}}}}}" + } + ], + "chartType": "pie", + "valueType": "number", + "action": "", + "documentType": "_doc", + "drillChart": "none", + "aggregationPaths": [ + "Complaints By Department" + ], + "insight": { + }, + "_comment": " Complaints by Department" + }, + "pgrStatusByDDR": { + "chartName": "DSS_PGR_STATUS_BY_DDR", + "queries": [ + { + "module": "PGR", + "requestQueryMap": "{\"tenantId\" : \"Data.tenantId.keyword\" , \"departmentId\" : \"Data.department.keyword\"}", + "dateRefField": "Data.dateOfComplaint", + "indexName": "pgrindex-v1", + "aggrQuery": "{\"aggs\":{\"AGGR\":{\"filter\":{\"bool\":{\"must_not\":[{\"term\":{\"Data.tenantId.keyword\":\"pb.testing\"}}]}},\"aggs\":{\"Closed Complaints\":{\"filters\":{\"filters\":{\"Open\":{\"match\":{\"Data.status.keyword\":\"open\"}},\"Reopen\":{\"match\":{\"Data.status.keyword\":\"reopen\"}},\"Assigned\":{\"match\":{\"Data.status.keyword\":\"assigned\"}},\"Closed\":{\"match\":{\"Data.status.keyword\":\"closed\"}},\"Resolved\":{\"match\":{\"Data.status.keyword\":\"resolved\"}},\"Rejected\":{\"match\":{\"Data.status.keyword\":\"rejected\"}},\"ReassignRequested\":{\"match\":{\"Data.status.keyword\":\"reassignrequested\"}}}}}}}}}" + } + ], + "isMdmsEnabled": true, + "filterKeys": [ + {"key": "tenantId", "column": "DDRs"} + ], + "chartType": "table", + "valueType": "number", + "drillChart": "pgrStatusByTenant", + "drillFields": [ + "" + ], + "documentType": "_doc", + "action": "", + "plotLabel": "DDRs", + "aggregationPaths": [ + "Open", + "Reopen", + "Assigned", + "Closed", + "Resolved", + "Rejected", + "ReassignRequested" + ], + "pathDataTypeMapping": [ + { + "Open": "number" + }, + { + "Reopen": "number" + }, + { + "Assigned": "number" + }, + { + "Closed": "number" + }, + { + "Resolved": "number" + }, + { + "Rejected": "number" + }, + { + "ReassignRequested": "number" + } + ], + "insight": { + }, + "_comment": "" + }, + "pgrStatusByTenant": { + "chartName": "DSS_PGR_STATUS_BY_TENANT", + "queries": [ + { + "module": "PGR", + "requestQueryMap": "{\"tenantId\" : \"Data.tenantId.keyword\" , \"departmentId\" : \"Data.department.keyword\"}", + "dateRefField": "Data.dateOfComplaint", + "indexName": "pgrindex-v1", + "aggrQuery": "{\"aggs\":{\"AGGR\":{\"filter\":{\"bool\":{\"must_not\":[{\"term\":{\"Data.tenantId.keyword\":\"pb.testing\"}}]}},\"aggs\":{\"ULBs \":{\"terms\":{\"field\":\"Data.tenantId.keyword\",\"size\":1000},\"aggs\":{\"Closed Complaints\":{\"filters\":{\"filters\":{\"Open\":{\"match\":{\"Data.status.keyword\":\"open\"}},\"Reopen\":{\"match\":{\"Data.status.keyword\":\"reopen\"}},\"Assigned\":{\"match\":{\"Data.status.keyword\":\"assigned\"}},\"Closed\":{\"match\":{\"Data.status.keyword\":\"closed\"}},\"Resolved\":{\"match\":{\"Data.status.keyword\":\"resolved\"}},\"Rejected\":{\"match\":{\"Data.status.keyword\":\"rejected\"}},\"ReassignRequested\":{\"match\":{\"Data.status.keyword\":\"reassignrequested\"}}}}}}}}}}}" + } + ], + "filterKeys": [ + {"key": "tenantId", "column": "Boundary"} + ], + "chartType": "table", + "valueType": "number", + "drillChart": "pgrStatusWardDrillDown", + "drillFields": [ + "" + ], + "documentType": "_doc", + "action": "", + "plotLabel": "Boundary", + "aggregationPaths": [ + "Open", + "Reopen", + "Assigned", + "Closed", + "Resolved", + "Rejected", + "ReassignRequested" + ], + "pathDataTypeMapping": [ + { + "Open": "number" + }, + { + "Reopen": "number" + }, + { + "Assigned": "number" + }, + { + "Closed": "number" + }, + { + "Resolved": "number" + }, + { + "Rejected": "number" + }, + { + "ReassignRequested": "number" + } + ], + "insight": { + }, + "_comment": "" + }, + "pgrStatusWardDrillDown": { + "chartName": "", + "queries": [ + { + "module": "PGR", + "requestQueryMap": "{\"tenantId\" : \"Data.tenantId.keyword\" , \"departmentId\" : \"Data.department.keyword\"}", + "dateRefField": "Data.dateOfComplaint", + "indexName": "pgrindex-v1", + "aggrQuery": "{\"aggs\":{\"AGGR\":{\"filter\":{\"bool\":{\"must_not\":[{\"term\":{\"Data.tenantId.keyword\":\"pb.testing\"}}]}},\"aggs\":{\"Ward \":{\"terms\":{\"field\":\"Data.complaintWard.name.keyword\",\"size\":1000},\"aggs\":{\"Closed Complaints\":{\"filters\":{\"filters\":{\"Open\":{\"match\":{\"Data.status.keyword\":\"open\"}},\"Reopen\":{\"match\":{\"Data.status.keyword\":\"reopen\"}},\"Assigned\":{\"match\":{\"Data.status.keyword\":\"assigned\"}},\"Closed\":{\"match\":{\"Data.status.keyword\":\"closed\"}},\"Resolved\":{\"match\":{\"Data.status.keyword\":\"resolved\"}},\"Rejected\":{\"match\":{\"Data.status.keyword\":\"rejected\"}},\"ReassignRequested\":{\"match\":{\"Data.status.keyword\":\"reassignrequested\"}}}}}}}}}}}" + } + ], + "filterKeys": [ + ], + "chartType": "table", + "valueType": "number", + "drillChart": "", + "drillFields": [ + "" + ], + "documentType": "_doc", + "action": "", + "plotLabel": "Complaint Ward", + "aggregationPaths": [ + "Open", + "Reopen", + "Assigned", + "Closed", + "Resolved", + "Rejected", + "ReassignRequested" + ], + "pathDataTypeMapping": [ + { + "Open": "number" + }, + { + "Reopen": "number" + }, + { + "Assigned": "number" + }, + { + "Closed": "number" + }, + { + "Resolved": "number" + }, + { + "Rejected": "number" + }, + { + "ReassignRequested": "number" + } + ], + "insight": { + }, + "_comment": "" + }, + "pgrStatusByDepartment": { + "chartName": "DSS_PGR_STATUS_BY_DEPARTMENT", + "queries": [ + { + "module": "PGR", + "requestQueryMap": "{\"tenantId\" : \"Data.tenantId.keyword\" }", + "dateRefField": "Data.dateOfComplaint", + "indexName": "pgrindex-v1", + "aggrQuery": "{\"aggs\":{\"AGGR\":{\"filter\":{\"bool\":{\"must_not\":[{\"term\":{\"Data.tenantId.keyword\":\"pb.testing\"}}]}},\"aggs\":{\"Department \":{\"terms\":{\"field\":\"Data.department.keyword\",\"size\":1000},\"aggs\":{\"Closed Complaints\":{\"filters\":{\"filters\":{\"Open\":{\"match\":{\"Data.status.keyword\":\"open\"}},\"Reopen\":{\"match\":{\"Data.status.keyword\":\"reopen\"}},\"Assigned\":{\"match\":{\"Data.status.keyword\":\"assigned\"}},\"Closed\":{\"match\":{\"Data.status.keyword\":\"closed\"}},\"Resolved\":{\"match\":{\"Data.status.keyword\":\"resolved\"}},\"Rejected\":{\"match\":{\"Data.status.keyword\":\"rejected\"}},\"ReassignRequested\":{\"match\":{\"Data.status.keyword\":\"reassignrequested\"}}}}}}}}}}}" + } + ], + "filterKeys": [ + {"key": "departmentId", "column": "Department"} + ], + "chartType": "table", + "valueType": "number", + "drillChart": "pgrStatusCategoryDrillDown", + "documentType": "_doc", + "action": "", + "plotLabel": "Department", + "aggregationPaths": [ + "Open", + "Reopen", + "Assigned", + "Closed", + "Resolved", + "Rejected", + "ReassignRequested" + ], + "pathDataTypeMapping": [ + { + "Open": "number" + }, + { + "Reopen": "number" + }, + { + "Assigned": "number" + }, + { + "Closed": "number" + }, + { + "Resolved": "number" + }, + { + "Rejected": "number" + }, + { + "ReassignRequested": "number" + } + ], + "insight": { + }, + "_comment": "" + }, + "pgrStatusCategoryDrillDown": { + "chartName": "", + "queries": [ + { + "module": "PGR", + "requestQueryMap": "{\"tenantId\" : \"Data.tenantId.keyword\" , \"departmentId\" : \"Data.department.keyword\"}", + "dateRefField": "Data.dateOfComplaint", + "indexName": "pgrindex-v1", + "aggrQuery": "{\"aggs\":{\"AGGR\":{\"filter\":{\"bool\":{\"must_not\":[{\"term\":{\"Data.tenantId.keyword\":\"pb.testing\"}}]}},\"aggs\":{\"Category \":{\"terms\":{\"field\":\"Data.complainCategory.keyword\",\"size\":1000},\"aggs\":{\"Closed Complaints\":{\"filters\":{\"filters\":{\"Open\":{\"match\":{\"Data.status.keyword\":\"open\"}},\"Reopen\":{\"match\":{\"Data.status.keyword\":\"reopen\"}},\"Assigned\":{\"match\":{\"Data.status.keyword\":\"assigned\"}},\"Closed\":{\"match\":{\"Data.status.keyword\":\"closed\"}},\"Resolved\":{\"match\":{\"Data.status.keyword\":\"resolved\"}},\"Rejected\":{\"match\":{\"Data.status.keyword\":\"rejected\"}},\"ReassignRequested\":{\"match\":{\"Data.status.keyword\":\"reassignrequested\"}}}}}}}}}}}" + } + ], + "filterKeys": [ + ], + "chartType": "table", + "valueType": "number", + "drillChart": "", + "drillFields": [ + "" + ], + "documentType": "_doc", + "action": "", + "plotLabel": "Complaint Category", + "aggregationPaths": [ + "Open", + "Reopen", + "Assigned", + "Closed", + "Resolved", + "Rejected", + "ReassignRequested" + ], + "pathDataTypeMapping": [ + { + "Open": "number" + }, + { + "Reopen": "number" + }, + { + "Assigned": "number" + }, + { + "Closed": "number" + }, + { + "Resolved": "number" + }, + { + "Rejected": "number" + }, + { + "ReassignRequested": "number" + } + ], + "insight": { + }, + "_comment": "" + }, + "topFiveComplaints": { + "chartName": "Top Complaints", + "queries": [ + { + "module": "PGR", + "requestQueryMap": "{\"tenantId\" : \"Data.tenantId\"}", + "dateRefField": "Data.dateOfComplaint", + "indexName": "pgrindex-v1", + "aggrQuery": "{\"aggs\":{\"AGGR\":{\"filter\":{\"bool\":{\"must_not\":[{\"term\":{\"Data.tenantId.keyword\":\"pb.testing\"}}]}},\"aggs\":{\"Complain Category\":{\"terms\":{\"field\":\"Data.complainCategory.keyword\",\"size\":5}}}}}}" + } + ], + "chartType": "pie", + "valueType": "number", + "action": "", + "documentType": "_doc", + "drillChart": "none", + "aggregationPaths": [ + "Complain Category" + ], + "insight": { + }, + "_comment": " Complaints By Their Statuses" + } +} diff --git a/dashboard-analytics/src/main/resources/schema/MasterDashboardConfig.json b/dashboard-analytics/src/main/resources/schema/MasterDashboardConfig.json new file mode 100644 index 000000000..c827ad736 --- /dev/null +++ b/dashboard-analytics/src/main/resources/schema/MasterDashboardConfig.json @@ -0,0 +1,1187 @@ +{ + "_comment": "Master dashboard which holds all the possible visualisation. Note new dashboards/visualisations can be added", + "dashboards": [ + { + "name": "State Urban Real-Time Executive (SURE) Dashboard", + "id": "home", + "isActive": "", + "style": "", + "visualizations": [ + { + "name": "DSS_REVENUE", + "vizArray": [ + { + "id": 111, + "name": "Overview", + "vizType": "metric-collection", + "noUnit": true, + "charts": [ + { + "id": "todaysCollection", + "name": "DSS_TOTAL_COLLECTION_TODAY", + "code": "", + "chartType": "", + "filter": "", + "headers": [] + }, + { + "id": "totalCollection", + "name": "DSS_TOTAL_COLLECTION", + "code": "", + "chartType": "metric", + "filter": "", + "headers": [] + }, + { + "id": "targetCollection", + "name": "DSS_TARGET_COLLECTION", + "code": "", + "chartType": "metric", + "filter": "", + "headers": [] + }, + { + "id": "targetAchieved", + "name": "DSS_TARGET_ACHIEVED", + "code": "", + "chartType": "metric", + "filter": "", + "headers": [] + } + ] + }, + { + "id": 112, + "name": "DSS_TOTAL_CUMULATIVE_COLLECTION", + "dimensions": { + "height": 350, + "width": 7 + }, + "vizType": "chart", + "label": "", + "noUnit": true, + "charts": [ + { + "id": "cumulativeCollection", + "name": "Weekly", + "code": "", + "chartType": "line", + "filter": "", + "headers": [] + } + ] + } + ] + }, + { + "row": 2, + "name": "DSS_REVENUE", + "vizArray": [ + { + "id": 121, + "name": "DSS_TOP_PERFORMING_ULBS", + "dimensions": { + "height": 250, + "width": 4 + }, + "vizType": "performing-metric", + "noUnit": false, + "label": "", + "charts": [ + { + "id": "topPerformingUlbs", + "name": "DSS_TOP_PERFORMING_ULBS", + "code": "", + "chartType": "bar", + "filter": "", + "headers": [] + } + ] + }, + { + "id": 122, + "name": "DSS_BOTTOM_PERFORMING_ULBS", + "dimensions": { + "height": 250, + "width": 4 + }, + "vizType": "performing-metric", + "noUnit": false, + "charts": [ + { + "id": "bottomPerformingUlbs", + "name": "DSS_BOTTOM_PERFORMING_ULBS", + "code": "", + "chartType": "bar", + "filter": "", + "headers": [] + } + ] + }, + { + "id": 123, + "name": "DSS_TOTAL_CUMULATIVE_COLLECTION:_DEPARTMENT_WISE", + "dimensions": { + "height": 250, + "width": 4 + }, + "vizType": "chart", + "noUnit": true, + "charts": [ + { + "id": "totalCollectionDeptWise", + "name": "DSS_TOTAL_CUMULATIVE_COLLECTION:_DEPARTMENT_WISE", + "code": "", + "chartType": "pie", + "filter": "", + "headers": [] + } + ] + } + ] + }, + { + "row": 3, + "name": "DSS_SERVICE", + "vizArray": [ + { + "id": 131, + "name": "Overview", + "dimensions": { + "height": 450, + "width": 5 + }, + "vizType": "metric-collection", + "label": "", + "charts": [ + { + "id": "totalApplication", + "name": "DSS_TOTAL_APPLICATION", + "code": "", + "chartType": "metric", + "filter": "", + "headers": [] + }, + { + "id": "closedApplication", + "name": "DSS_CLOSED_APPLICATION", + "code": "", + "chartType": "metric", + "filter": "", + "headers": [] + }, + { + "id": "slaAchieved", + "name": "DSS_SLA_ACHIEVED", + "code": "", + "chartType": "metric", + "filter": "", + "headers": [] + }, + { + "id": "citizenRegistered", + "name": "DSS_CITIZEN_REGISTERED", + "code": "", + "chartType": "metric", + "filter": "", + "headers": [] + } + ] + }, + { + "id": 132, + "name": "DSS_TOTAL_APPLICATION_&_CLOSED_APPLICATION", + "dimensions": { + "height": 450, + "width": 7 + }, + "vizType": "chart", + "charts": [ + { + "id": "totalApplication&ClosedApplication", + "name": "DSS_TOTAL_APPLICATION_&_CLOSED_APPLICATION", + "code": "", + "chartType": "line", + "filter": "", + "headers": [] + } + ] + } + ] + }, + { + "row": 4, + "name": "DSS_SERVICE", + "vizArray": [ + { + "id": 141, + "name": "DSS_TOP_PERFORMING_ULBS", + "dimensions": { + "height": 250, + "width": 4 + }, + "vizType": "performing-metric", + "label": "", + "charts": [ + { + "id": "topPerformingUlbsCompletionRate", + "name": "DSS_TOP_PERFORMING_ULBS", + "code": "", + "chartType": "bar", + "filter": "", + "headers": [] + } + ] + }, + { + "id": 142, + "name": "DSS_BOTTOM_PERFORMING_ULBS", + "dimensions": { + "height": 250, + "width": 4 + }, + "vizType": "performing-metric", + "charts": [ + { + "id": "bottomPerformingUlbsCompletionRate", + "name": "DSS_BOTTOM_PERFORMING_ULBS", + "code": "", + "chartType": "bar", + "filter": "", + "headers": [] + } + ] + }, + { + "id": 143, + "name": "DSS_TOTAL_APPLICATIONS:_DEPARTMENT_WISE", + "dimensions": { + "height": 250, + "width": 4 + }, + "vizType": "chart", + "charts": [ + { + "id": "totalApplicationDeptWise", + "name": "DSS_TOTAL_APPLICATIONS:_DEPARTMENT_WISE", + "code": "", + "chartType": "pie", + "filter": "", + "headers": [] + } + ] + } + ] + } + ] + }, + { + "name": "Property Tax (SURE) Dashboard", + "id": "propertyTax", + "isActive": "", + "style": "linear", + "visualizations": [ + { + "row": 1, + "name": "DSS_REVENUE", + "vizArray": [ + { + "id": 211, + "name": "Overview", + "dimensions": { + "height": 350, + "width": 5 + }, + "vizType": "metric-collection", + "noUnit": true, + "label": "Overview", + "charts": [ + { + "id": "todaysCollectionPT", + "name": "DSS_TOTAL_COLLECTION_TODAY", + "code": "", + "chartType": "metric", + "filter": "", + "headers": [] + }, + { + "id": "totalCollection", + "name": "DSS_TOTAL_COLLECTION", + "code": "", + "chartType": "metric", + "filter": "", + "headers": [] + }, + { + "id": "targetCollection", + "name": "DSS_TARGET_COLLECTION", + "code": "", + "chartType": "metric", + "filter": "", + "headers": [] + }, + { + "id": "targetAchieved", + "name": "DSS_TARGET_ACHIEVED", + "code": "", + "chartType": "metric", + "filter": "", + "headers": [] + } + ] + }, + { + "id": 212, + "name": "DSS_TOTAL_CUMULATIVE_COLLECTION", + "dimensions": { + "height": 350, + "width": 7 + }, + "vizType": "chart", + "noUnit": true, + "charts": [ + { + "id": "cumulativeCollection", + "name": "Monthly", + "code": "", + "chartType": "line", + "filter": "", + "headers": [] + } + ] + } + ] + }, + { + "row": 2, + "name": "DSS_REVENUE", + "vizArray": [ + { + "id": 221, + "name": "DSS_PT_TOP_3_PERFORMING_ULBS", + "dimensions": { + "height": 250, + "width": 4 + }, + "vizType": "performing-metric", + "noUnit": false, + "label": "", + "charts": [ + { + "id": "topPerformingUlbs", + "name": "DSS_PT_TOP_3_PERFORMING_ULBS", + "code": "", + "chartType": "bar", + "filter": "", + "headers": [] + } + ] + }, + { + "id": 222, + "name": "DSS_PT_BOTTOM_3_PERFORMING_ULBS", + "dimensions": { + "height": 250, + "width": 4 + }, + "vizType": "performing-metric", + "noUnit": false, + "charts": [ + { + "id": "bottomPerformingUlbs", + "name": "DSS_PT_BOTTOM_3_PERFORMING_ULBS", + "code": "", + "chartType": "bar", + "filter": "", + "headers": [] + } + ] + }, + { + "id": 223, + "name": "DSS_PT_COLLECTION_BY_USAGE_TYPE", + "dimensions": { + "height": 250, + "width": 4 + }, + "vizType": "chart", + "noUnit": true, + "charts": [ + { + "id": "collectionByUsageType", + "name": "DSS_PT_COLLECTION_BY_USAGE_TYPE", + "code": "", + "chartType": "pie", + "filter": "", + "headers": [] + } + ] + } + ] + }, + { + "row": 3, + "name": "DSS_REVENUE", + "vizArray": [ + { + "id": 231, + "name": "DSS_PT_DEMAND_COLLECTION", + "dimensions": { + "height": 350, + "width": 12 + }, + "vizType": "chart", + "noUnit": false, + "charts": [ + { + "id": "demandCollectionIndexDDRRevenue", + "name": "DSS_PT_DEMAND_COLLECTION_BOUNDARY", + "code": "", + "chartType": "table", + "filter": "", + "headers": [], + "tabName": "Boundary" + }, + { + "id": "demandCollectionIndexUsageRevenue", + "name": "DSS_PT_DEMAND_COLLECTION_USAGETYPE", + "code": "", + "chartType": "table", + "filter": "", + "headers": [], + "tabName": "Usage" + } + ] + } + ] + }, + { + "row": 4, + "name": "DSS_REVENUE", + "vizArray": [ + { + "id": 231, + "name": "DSS_PT_TAX_HEAD", + "dimensions": { + "height": 350, + "width": 12 + }, + "vizType": "chart", + "label": "", + "noUnit": false, + "charts": [ + { + "id": "taxHeadsBreakupDDRRevenue", + "name": "DSS_PT_TAX_HEAD_BREAKUP_REVENUE", + "code": "", + "chartType": "table", + "filter": "", + "headers": [], + "tabName": "Boundary" + + }, + { + "id": "taxHeadsBreakupUsage", + "name": "DSS_PT_TAX_HEAD_BREAKUP_USAGE", + "code": "", + "chartType": "table", + "filter": "", + "headers": [], + "tabName": "Usage" + + } + ] + } + ] + }, + { + "row": 5, + "name": "DSS_SERVICE", + "vizArray": [ + { + "id": 241, + "name": "Overview", + "dimensions": { + "height": 350, + "width": 5 + }, + "vizType": "metric-collection", + "label": "Overview", + "charts": [ + { + "id": "propertiesAssessed", + "name": "DSS_PT_TOTAL_PROPERTIES_ASSESSED", + "code": "", + "chartType": "metric", + "filter": "", + "headers": [] + }, + { + "id": "propertiesAssessed", + "name": "DSS_PT_TOTAL_ASSESSMENTS", + "code": "", + "chartType": "metric", + "filter": "", + "headers": [] + }, + { + "id": "activeUlbs", + "name": "DSS_PT_TOTAL_ACTIVE_ULBS", + "code": "", + "chartType": "metric", + "filter": "", + "headers": [] + } + ] + }, + { + "id": 242, + "name": "DSS_PT_CUMULATIVE_PROPERTIES_ASSESSED", + "dimensions": { + "height": 350, + "width": 7 + }, + "vizType": "chart", + "charts": [ + { + "id": "cumulativePropertiesAssessed", + "name": "Monthly", + "code": "", + "chartType": "line", + "filter": "", + "headers": [] + } + ] + } + ] + }, + { + "row": 6, + "name": "DSS_SERVICE", + "vizArray": [ + { + "id": 251, + "name": "DSS_PT_TOP_3_PERFORMING_ULBS", + "dimensions": { + "height": 250, + "width": 4 + }, + "vizType": "performing-metric", + "label": "", + "charts": [ + { + "id": "topPerformingUlbsCompletionRate", + "name": "DSS_PT_TOP_3_PERFORMING_ULBS", + "code": "", + "chartType": "bar", + "filter": "", + "headers": [] + } + ] + }, + { + "id": 252, + "name": "DSS_PT_BOTTOM_3_PERFORMING_ULBS", + "dimensions": { + "height": 250, + "width": 4 + }, + "vizType": "performing-metric", + "charts": [ + { + "id": "bottomPerformingUlbsCompletionRate", + "name": "DSS_PT_BOTTOM_3_PERFORMING_ULBS", + "code": "", + "chartType": "bar", + "filter": "", + "headers": [] + } + ] + }, + { + "id": 253, + "name": "DSS_PT_PROPERTIES_BY_USAGE_TYPE", + "dimensions": { + "height": 250, + "width": 4 + }, + "vizType": "chart", + "charts": [ + { + "id": "propertiesByUsageType", + "name": "DSS_PT_PROPERTIES_BY_USAGE_TYPE", + "code": "", + "chartType": "pie", + "filter": "", + "headers": [] + } + ] + } + ] + } + ] + }, + { + "name": "Trade License (SURE) Dashboard", + "id": "tradeLicense", + "isActive": "", + "style": "linear", + "visualizations": [ + { + "row": 1, + "name": "DSS_REVENUE", + "vizArray": [ + { + "name": "Overview", + "dimensions": { + "height": 350, + "width": 5 + }, + "vizType": "metric-collection", + "label": "Overview", + "noUnit": true, + "charts": [ + { + "id": "todaysCollectionTL", + "name": "DSS_TOTAL_COLLECTION_TODAY", + "code": "", + "chartType": "metric", + "filter": "", + "headers": [] + }, + { + "id": "totalCollection", + "name": "DSS_TOTAL_COLLECTION", + "code": "", + "chartType": "metric", + "filter": "", + "headers": [] + }, + { + "id": "targetCollection", + "name": "DSS_TARGET_COLLECTION", + "code": "", + "chartType": "metric", + "filter": "", + "headers": [] + }, + { + "id": "targetAchieved", + "name": "DSS_TARGET_ACHIEVED", + "code": "", + "chartType": "metric", + "filter": "", + "headers": [] + } + ] + }, + { + "id": 312, + "name": "Total Cumulative Collection", + "dimensions": { + "height": 350, + "width": 7 + }, + "vizType": "chart", + "label": "", + "noUnit": true, + "charts": [ + { + "id": "cumulativeCollection", + "name": "Monthly", + "code": "", + "chartType": "line", + "filter": "", + "headers": [] + } + ] + } + ] + }, + { + "row": 2, + "name": "DSS_REVENUE", + "vizArray": [ + { + "id": 321, + "name": "DSS_PT_TOP_3_PERFORMING_ULBS", + "dimensions": { + "height": 250, + "width": 4 + }, + "vizType": "performing-metric", + "label": "", + "noUnit": false, + "charts": [ + { + "id": "topPerformingUlbs", + "name": "DSS_PT_TOP_3_PERFORMING_ULBS", + "code": "", + "chartType": "bar", + "filter": "", + "headers": [] + } + ] + }, + { + "id": 322, + "name": "DSS_PT_BOTTOM_3_PERFORMING_ULBS", + "dimensions": { + "height": 250, + "width": 4 + }, + "vizType": "performing-metric", + "label": "", + "noUnit": false, + "charts": [ + { + "id": "bottomPerformingUlbs", + "name": "DSS_PT_BOTTOM_3_PERFORMING_ULBS", + "code": "", + "chartType": "bar", + "filter": "", + "headers": [] + } + ] + }, + { + "id": 323, + "name": "DSS_TL_LICENSE_BY_TYPE", + "dimensions": { + "height": 250, + "width": 4 + }, + "vizType": "chart", + "label": "", + "noUnit": false, + "charts": [ + { + "id": "licenseByLicenseType", + "name": "DSS_TL_LICENSE_BY_TYPE", + "code": "", + "chartType": "pie", + "filter": "", + "headers": [] + } + ] + } + ] + }, + { + "row": 4, + "name": "DSS_REVENUE", + "vizArray": [ + { + "id": 231, + "name": "DSS_TL_DEMAND_COLLECTION", + "dimensions": { + "height": 350, + "width": 12 + }, + "vizType": "chart", + "label": "", + "noUnit": false, + "charts": [ + { + "id": "licenseIssuedDDRRevenue", + "name": "DSS_TL_DEMAND_COLLECTION_BOUNDARY", + "code": "", + "chartType": "table", + "filter": "", + "headers": [], + "tabName": "Boundary" + + } + ] + } + ] + }, + { + "row": 5, + "name": "DSS_REVENUE", + "vizArray": [ + { + "id": 231, + "name": "DSS_TL_TAX_HEAD_BREAKUP", + "dimensions": { + "height": 350, + "width": 12 + }, + "vizType": "chart", + "label": "", + "noUnit": false, + "charts": [ + { + "id": "licenceTaxHeadsBreakupDDR", + "name": "DSS_TL_TAX_HEAD_BREAKUP_BOUNDARY", + "code": "", + "chartType": "table", + "filter": "", + "headers": [], + "tabName": "Boundary" + + } + ] + } + ] + }, + { + "row": 6, + "name": "DSS_SERVICE", + "vizArray": [ + { + "id": 341, + "name": "Overview", + "dimensions": { + "height": 350, + "width": 5 + }, + "vizType": "metric-collection", + "label": "Overview", + "charts": [ + { + "id": "totalApplication", + "name": "DSS_TOTAL_APPLICATION", + "code": "", + "chartType": "metric", + "filter": "", + "headers": [] + }, + { + "id": "licenseIssued", + "name": "DSS_TL_LICENSE_ISSUED", + "code": "", + "chartType": "metric", + "filter": "", + "headers": [] + }, + { + "id": "activeUlbs", + "name": "DSS_PT_TOTAL_ACTIVE_ULBS", + "code": "", + "chartType": "metric", + "filter": "", + "headers": [] + } + ] + }, + { + "id": 342, + "name": "DSS_TL_CUMULATIVE_LICENSE_ISSUED", + "dimensions": { + "height": 350, + "width": 7 + }, + "vizType": "chart", + "charts": [ + { + "id": "cumulativeLicenseIssued", + "name": "Monthly", + "code": "", + "chartType": "line", + "filter": "", + "headers": [] + } + ] + } + ] + }, + { + "row": 7, + "name": "DSS_SERVICE", + "vizArray": [ + { + "id": 351, + "name": "DSS_PT_TOP_3_PERFORMING_ULBS", + "dimensions": { + "height": 250, + "width": 4 + }, + "vizType": "performing-metric", + "label": "", + "noUnit": false, + "charts": [ + { + "id": "topPerformingUlbsCompletionRate", + "name": "DSS_PT_TOP_3_PERFORMING_ULBS", + "code": "", + "chartType": "bar", + "filter": "", + "headers": [] + } + ] + }, + { + "id": 352, + "name": "DSS_PT_BOTTOM_3_PERFORMING_ULBS", + "dimensions": { + "height": 250, + "width": 4 + }, + "vizType": "performing-metric", + "noUnit": false, + "charts": [ + { + "id": "bottomPerformingUlbsCompletionRate", + "name": "DSS_PT_BOTTOM_3_PERFORMING_ULBS", + "code": "", + "chartType": "bar", + "filter": "", + "headers": [] + } + ] + }, + { + "id": 353, + "name": "DSS_TL_LICENSE_BY_STATUS", + "dimensions": { + "height": 250, + "width": 4 + }, + "vizType": "chart", + "charts": [ + { + "id": "licenseApplicationByStatus", + "name": "DSS_TL_LICENSE_BY_STATUS", + "code": "", + "chartType": "pie", + "filter": "", + "headers": [] + } + ] + } + ] + } + ] + }, + { + "name": "PGR (SURE) Dashboard", + "id": "pgr", + "isActive": "", + "style": "linear", + "visualizations": [ + + { + "row": 1, + "name": "DSS_SERVICE", + "vizArray": [ + { + "id": 412, + "name": "Overview", + "dimensions": { + "height": 350, + "width": 5 + }, + "vizType": "metric-collection", + "label": "Overview", + "noUnit": true, + "charts": [ + { + "id": "totalComplaints", + "name": "DSS_PGR_TOTAL_COMPLAINTS", + "code": "", + "chartType": "metric", + "filter": "", + "headers": [] + }, + { + "id": "closedComplaints", + "name": "DSS_PGR_CLOSED_COMPLAINTS", + "code": "", + "chartType": "metric", + "filter": "", + "headers": [] + }, + { + "id": "slaAchieved", + "name": "DSS_SLA_ACHIEVED", + "code": "", + "chartType": "metric", + "filter": "", + "headers": [] + }, + { + "id": "completionRate", + "name": "DSS_PGR_COMPLETION_RATE", + "code": "", + "chartType": "metric", + "filter": "", + "headers": [] + } + ] + }, + { + "id": 412, + "name": "DSS_TOTAL_CUMULATIVE_CLOSED_COMPLAINTS", + "dimensions": { + "height": 350, + "width": 7 + }, + "vizType": "chart", + "label": "", + "charts": [ + { + "id": "cumulativeClosedComplaints", + "name": "Monthly", + "code": "", + "chartType": "line", + "filter": "", + "headers": [] + } + ] + } + ] + }, + { + "row": 2, + "name": "DSS_SERVICE", + "vizArray": [ + { + "id": 412, + "name": "DSS_TOTAL_COMPLAINTS_BY_SOURCE", + "dimensions": { + "height": 350, + "width": 7 + }, + "vizType": "chart", + "label": "", + "charts": [ + { + "id": "totalComplaintsbySource", + "name": "Monthly", + "code": "", + "chartType": "line", + "filter": "", + "headers": [] + } + ] + }, + { + "id": 413, + "name": "DSS_TOTAL_COMPLAINTS_STATUS", + "dimensions": { + "height": 350, + "width": 7 + }, + "vizType": "chart", + "label": "", + "charts": [ + { + "id": "totalComplaintsbyStatus", + "name": "", + "code": "", + "chartType": "line", + "filter": "", + "headers": [] + } + ] + } + ] + }, + { + "row": 3, + "name": "DSS_SERVICE", + "vizArray": [ + { + "id": 421, + "name": "DSS_PGR_COMPLAINTS_BY_STATUS", + "dimensions": { + "height": 250, + "width": 4 + }, + "vizType": "chart", + "label": "", + "charts": [ + { + "id": "complaintsByStatus", + "name": "DSS_PGR_COMPLAINTS_BY_STATUS", + "code": "", + "chartType": "pie", + "filter": "", + "headers": [] + } + ] + }, + { + "id": 422, + "name": "DSS_PGR_COMPLAINTS_BY_CHANNEL", + "dimensions": { + "height": 250, + "width": 4 + }, + "vizType": "chart", + "charts": [ + { + "id": "complaintsByChannel", + "name": "DSS_PGR_COMPLAINTS_BY_CHANNEL", + "code": "", + "chartType": "pie", + "filter": "", + "headers": [] + } + ] + }, + { + "id": 423, + "name": "DSS_PGR_COMPLAINTS_BY_DEPARTMENT", + "dimensions": { + "height": 250, + "width": 4 + }, + "vizType": "chart", + "charts": [ + { + "id": "complaintsByDepartment", + "name": "DSS_PGR_COMPLAINTS_BY_DEPARTMENT", + "code": "", + "chartType": "pie", + "filter": "", + "headers": [] + } + ] + } + ] + }, + { + "row": 4, + "name": "DSS_SERVICE", + "vizArray": [ + { + "id": 431, + "name": "DSS_PGR_STATUS_BY_TENANT", + "dimensions": { + "height": 350, + "width": 12 + }, + "vizType": "chart", + "label": "", + "noUnit": false, + "charts": [ + { + "id": "pgrStatusByTenant", + "name": "DSS_PGR_STATUS_BY_TENANT", + "code": "", + "chartType": "table", + "filter": "", + "headers": [], + "tabName": "Boundary" + + }, + { + "id": "pgrStatusByDepartment", + "name": "DSS_PGR_STATUS_BY_DEPARTMENT", + "code": "", + "chartType": "table", + "filter": "", + "headers": [], + "tabName": "Department" + + } + ] + } + ] + } + ] + } + ] +} \ No newline at end of file diff --git a/dashboard-analytics/src/main/resources/schema/RoleDashboardConfig.json b/dashboard-analytics/src/main/resources/schema/RoleDashboardConfig.json new file mode 100644 index 000000000..df88cb557 --- /dev/null +++ b/dashboard-analytics/src/main/resources/schema/RoleDashboardConfig.json @@ -0,0 +1,562 @@ +{ + "_comment": "TODO: denormalize the role & visualisations", + "roles" : [ + { + "roleId": 6, + "roleName" : "R1", + "isSuper" : "", + "orgId": "", + "dashboards": [ + { + "name": "My Dashboard", + "id": "home", + "isActive": "", + "style" : "tabbed", + "visualizations": [ + { + "row": 1, + "name": "Revenue", + "vizArray": [ + { + "id": 11, + "name": "", + "dimensions": { + "height": 350, + "width": 5 + }, + "vizType": "metric-collection", + "charts": [ + { + "id": "totalCollection", + "name": "Total Collection", + "code": "", + "chartType": "metric", + "filter": "", + "headers": [] + }, + { + "id": "targetCollection", + "name": "Target Collection", + "code": "", + "chartType": "metric", + "filter": "", + "headers": [] + }, + { + "id": "targetAchieved", + "name": "Target Achieved", + "code": "", + "chartType": "metric", + "filter": "", + "headers": [] + } + ] + }, + { + "id": 12, + "name": "Total Cumulative Collection", + "dimensions": { + "height": 350, + "width": 7 + }, + "vizType": "chart", + "charts": [ + { + "id": "cumulativeCollection", + "name": "Weekly", + "code": "", + "chartType": "line", + "filter": "", + "headers": [] + } + ] + } + ] + }, + { + "row": 2, + "name": "Revenue", + "vizArray": [ + { + "id": 21, + "name": "Top Performing ULBs", + "dimensions": { + "height": 250, + "width": 4 + }, + "vizType": "performing-metric", + "charts": [ + { + "id": "topPerformingUlbs", + "name": "Top Performing ULBs", + "code": "", + "chartType": "bar", + "filter": "", + "headers": [] + } + ] + }, + { + "id": 22, + "name": "Bottom Performing ULBs", + "dimensions": { + "height": 250, + "width": 4 + }, + "vizType": "performing-metric", + "charts": [ + { + "id": "bottomPerformingUlbs", + "name": "Bottom Performing ULBs", + "code": "", + "chartType": "bar", + "filter": "", + "headers": [] + } + ] + }, + { + "id": 23, + "name": "Total Cumulative Collection: Department Wise", + "dimensions": { + "height": 250, + "width": 4 + }, + "vizType": "chart", + "charts": [ + { + "id": "totalCollectionDeptWise", + "name": "Total Cumulative Collection: Department Wise", + "code": "", + "chartType": "pie", + "filter": "", + "headers": [] + } + ] + } + ] + }, + { + "row": 3, + "name": "Service", + "vizArray": [ + { + "id": 31, + "name": "", + "dimensions": { + "height": 450, + "width": 5 + }, + "vizType": "metric-collection", + "charts": [ + { + "id": "totalApplication", + "name": "Total Application", + "code": "", + "chartType": "metric", + "filter": "", + "headers": [] + }, + { + "id": "closedApplication", + "name": "Closed Application", + "code": "", + "chartType": "metric", + "filter": "", + "headers": [] + }, + { + "id": "", + "name": "SLA Achieved", + "code": "", + "chartType": "metric", + "filter": "", + "headers": [] + }, + { + "id": "citizenRegistered", + "name": "Citizen Registered", + "code": "", + "chartType": "metric", + "filter": "", + "headers": [] + } + ] + }, + { + "id": 32, + "name": "Total Application & Closed Application", + "dimensions": { + "height": 450, + "width": 7 + }, + "vizType": "chart", + "charts": [ + { + "id": "totalApplication&ClosedApplication", + "name": "", + "code": "", + "chartType": "line", + "filter": "", + "headers": [] + } + ] + } + ] + }, + { + "row": 4, + "name": "Service", + "vizArray": [ + { + "id": 41, + "name": "Top Performing ULBs", + "dimensions": { + "height": 250, + "width": 4 + }, + "vizType": "performing-metric", + "charts": [ + { + "id": "topPerformingUlbsCompletionRate", + "name": "Top Performing ULBs", + "code": "", + "chartType": "bar", + "filter": "", + "headers": [] + } + ] + }, + { + "id": 42, + "name": "Bottom Performing ULBs", + "dimensions": { + "height": 250, + "width": 4 + }, + "vizType": "performing-metric", + "charts": [ + { + "id": "bottomPerformingUlbsCompletionRate", + "name": "Bottom Performing ULBs", + "code": "", + "chartType": "bar", + "filter": "", + "headers": [] + } + ] + }, + { + "id": 43, + "name": "Total Applications: Department Wise", + "dimensions": { + "height": 250, + "width": 4 + }, + "vizType": "chart", + "charts": [ + { + "id": "totalApplicationDeptWise", + "name": "Total Cumulative Collection: Department Wise", + "code": "", + "chartType": "pie", + "filter": "", + "headers": [] + } + ] + } + ] + } + ] + }, + { + "name": "Property Tax", + "id": "propertyTax", + "isActive":"", + "style" : "linear", + "visualizations": [ + { + "row": 1, + "name": "Revenue", + "vizArray": [ + { + "id": 11, + "name": "", + "dimensions": { + "height": 350, + "width": 5 + }, + "vizType": "metric-collection", + "charts": [ + { + "id": "totalCollection", + "name": "Total Collection", + "code": "", + "chartType": "metric", + "filter": "", + "headers": [] + }, + { + "id": "targetCollection", + "name": "Target Collection", + "code": "", + "chartType": "metric", + "filter": "", + "headers": [] + }, + { + "id": "targetAchieved", + "name": "Target Achieved", + "code": "", + "chartType": "metric", + "filter": "", + "headers": [] + } + ] + }, + { + "id": 12, + "name": "Total Cumulative Collection", + "dimensions": { + "height": 350, + "width": 7 + }, + "vizType": "chart", + "charts": [ + { + "id": "cumulativeCollection", + "name": "Monthly", + "code": "", + "chartType": "line", + "filter": "", + "headers": [] + } + ] + } + ] + }, + { + "row": 2, + "name": "Revenue", + "vizArray": [ + { + "id": 21, + "name": "Top 3 Performing ULBs", + "dimensions": { + "height": 250, + "width": 4 + }, + "vizType": "performing-metric", + "charts": [ + { + "id": "topPerformingUlbs", + "name": "Top 3 Performing ULBs", + "code": "", + "chartType": "bar", + "filter": "", + "headers": [] + } + ] + }, + { + "id": 22, + "name": "Bottom 3 Performing ULBs", + "dimensions": { + "height": 250, + "width": 4 + }, + "vizType": "performing-metric", + "charts": [ + { + "id": "bottomPerformingUlbs", + "name": "Bottom 3 Performing ULBs", + "code": "", + "chartType": "bar", + "filter": "", + "headers": [] + } + ] + }, + { + "id": 23, + "name": "Collection by Usage type", + "dimensions": { + "height": 250, + "width": 4 + }, + "vizType": "chart", + "charts": [ + { + "id": "collectionByUsageType", + "name": "Collection by Usage type", + "code": "", + "chartType": "pie", + "filter": "", + "headers": [] + } + ] + } + ] + }, + { + "row": 3, + "name": "Revenue", + "vizArray": [ + { + "id": 31, + "name": "", + "dimensions": { + "height": 350, + "width": 12 + }, + "vizType": "chart", + "charts": [ + { + "id": "demandCollectionIndexBoundaryRevenue", + "name": "Demand & Collection Index", + "code": "", + "chartType": "table", + "filter": "", + "headers": [] + }, + { + "id": "demandCollectionIndexUsageRevenue", + "name": "Demand & Collection Index", + "code": "", + "chartType": "table", + "filter": "", + "headers": [] + } + ] + } + ] + }, + { + "row": 4, + "name": "Service", + "vizArray": [ + { + "id": 41, + "name": "", + "dimensions": { + "height": 350, + "width": 5 + }, + "vizType": "metric-collection", + "charts": [ + { + "id": "propertiesAssessed", + "name": "Total Properties Assessed", + "code": "", + "chartType": "metric", + "filter": "", + "headers": [] + }, + { + "id": "propertiesAssessed", + "name": "Total Assessments", + "code": "", + "chartType": "metric", + "filter": "", + "headers": [] + }, + { + "id": "activeUlbs", + "name": "Total Active ULBs", + "code": "", + "chartType": "metric", + "filter": "", + "headers": [] + } + ] + }, + { + "id": 42, + "name": "Total Properties Assessed", + "dimensions": { + "height": 350, + "width": 7 + }, + "vizType": "chart", + "charts": [ + { + "id": "cumulativePropertiesAssessed", + "name": "Monthly", + "code": "", + "chartType": "line", + "filter": "", + "headers": [] + } + ] + } + ] + }, + { + "row": 5, + "name": "Service", + "vizArray": [ + { + "id": 21, + "name": "Top 3 Performing ULBs", + "dimensions": { + "height": 250, + "width": 4 + }, + "vizType": "performing-metric", + "charts": [ + { + "id": "topPerformingUlbsCompletionRate", + "name": "Top 3 Performing ULBs", + "code": "", + "chartType": "bar", + "filter": "", + "headers": [] + } + ] + }, + { + "id": 22, + "name": "Bottom 3 Performing ULBs", + "dimensions": { + "height": 250, + "width": 4 + }, + "vizType": "performing-metric", + "charts": [ + { + "id": "bottomPerformingUlbsCompletionRate", + "name": "Bottom 3 Performing ULBs", + "code": "", + "chartType": "bar", + "filter": "", + "headers": [] + } + ] + }, + { + "id": 23, + "name": "Properties by Usage type", + "dimensions": { + "height": 250, + "width": 4 + }, + "vizType": "chart", + "charts": [ + { + "id": "propertiesByUsageType", + "name": "Properties by Usage type", + "code": "", + "chartType": "pie", + "filter": "", + "headers": [] + } + ] + } + ] + } + ] + } + ] + } + ] +} diff --git a/dashboard-analytics/src/main/resources/schema/RoleDashboardMappingsConf.json b/dashboard-analytics/src/main/resources/schema/RoleDashboardMappingsConf.json new file mode 100644 index 000000000..f77bf640b --- /dev/null +++ b/dashboard-analytics/src/main/resources/schema/RoleDashboardMappingsConf.json @@ -0,0 +1,31 @@ +{ + "_comment": "Holds mapping for each role with and its associated dashboards", + "roles" : [ + { + "_comment":"This role is super role which can access all the available dashboards: [other/new roles are suppose to be added]", + "roleId": 6, + "roleName" : "Admin", + "isSuper" : "", + "orgId": "", + "dashboards": [ + { + "name": "My Dashboard", + "id": "home" + }, + { + "name": "Property Tax", + "id": "propertyTax" + }, + { + "name": "Trade License", + "id": "tradeLicense" + }, + { + "name": "PGR", + "id": "pgr" + } + ] + } + + ] +} \ No newline at end of file diff --git a/dashboard-analytics/src/main/resources/schema/getchart.json b/dashboard-analytics/src/main/resources/schema/getchart.json new file mode 100644 index 000000000..9f9e5ef89 --- /dev/null +++ b/dashboard-analytics/src/main/resources/schema/getchart.json @@ -0,0 +1,43 @@ +{ + "$schema": "ain/getchart/schema1#", + "title": "getchart", + "description": "getchart api schema", + "type": "object", + "properties": { + "header": { + "type": "object", + "properties": { + "tenantId": { + "type": "string" + } + } + }, + "requestInfo": { + "type": "object", + "properties": { + "chartType":{ + "type":"string", + "enum":["bar","line","stackedbar","pie", "horizontalBar", "doughnut", "doubledonut","heat","radar"] + }, + "chartFormat":{ + "type":"string" + }, + "serviceApi":{ + "type":"string" + }, + "dates":{ + "type":"object", + "properties":{ + "startDate":{ + "type":"string" + }, + "endData":{ + "type":"string" + } + } + } + }, + "required": ["chartType","chartFormat", "serviceApi","dates"] + } + } +} \ No newline at end of file diff --git a/dashboard-ingest/pom.xml b/dashboard-ingest/pom.xml new file mode 100644 index 000000000..61947ce89 --- /dev/null +++ b/dashboard-ingest/pom.xml @@ -0,0 +1,195 @@ + + 4.0.0 + + com.ingestpipeline + ingestpipeline + 0.0.1-SNAPSHOT + jar + + ingestpipeline + + + org.springframework.boot + spring-boot-starter-parent + 2.1.8.RELEASE + + + + 1.8 + UTF-8 + UTF-8 + 1.8 + 3.0.1 + + + + + org.springframework.boot + spring-boot-starter-web + + + com.fasterxml.jackson.dataformat + jackson-dataformat-xml + + + org.springframework.kafka + spring-kafka + + + + + com.google.code.gson + gson + 2.8.0 + + + com.bazaarvoice.jolt + jolt-core + 0.1.1 + + + com.bazaarvoice.jolt + json-utils + 0.1.1 + + + javax.ws.rs + jsr311-api + 1.1.1 + + + org.apache.commons + commons-lang3 + 3.4 + + + commons-io + commons-io + 2.6 + + + org.everit.json + org.everit.json.schema + 1.3.0 + + + org.elasticsearch + elasticsearch + 6.6.0 + + + org.elasticsearch.client + elasticsearch-rest-high-level-client + 6.6.0 + + + commons-fileupload + commons-fileupload + 1.4 + + + org.apache.poi + poi + 4.1.0 + + + org.apache.poi + poi-ooxml + 4.1.0 + + + + commons-beanutils + commons-beanutils + 1.8.3 + + + net.sf.ezmorph + ezmorph + 1.0.6 + + + commons-collections + commons-collections + 3.2.1 + + + commons-lang + commons-lang + 2.6 + + + org.json + json + 20160810 + + + + + com.fasterxml.jackson.core + jackson-databind + 2.9.8 + + + + org.springframework.boot + spring-boot-starter-data-jpa + 2.1.4.RELEASE + + + com.h2database + h2 + runtime + 1.4.199 + + + org.apache.commons + commons-lang3 + 3.7 + + + + com.github.wnameless + json-flattener + 0.2.2 + + + + + + + org.springframework.boot + spring-boot-starter-jdbc + + + org.springframework + spring-jdbc + + + org.json + json + 20190722 + + + + com.google.guava + guava + 20.0 + + + + + + + org.springframework.boot + spring-boot-maven-plugin + + + + diff --git a/dashboard-ingest/src/main/java/com/ingestpipeline/AppStartupRunner.java b/dashboard-ingest/src/main/java/com/ingestpipeline/AppStartupRunner.java new file mode 100644 index 000000000..bdcf3bbdb --- /dev/null +++ b/dashboard-ingest/src/main/java/com/ingestpipeline/AppStartupRunner.java @@ -0,0 +1,52 @@ +package com.ingestpipeline; + +import com.ingestpipeline.service.IngestServiceImpl; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.ApplicationArguments; +import org.springframework.boot.ApplicationRunner; +import org.springframework.stereotype.Component; + +import com.ingestpipeline.config.DomainConfig; +import com.ingestpipeline.config.DomainConfigFactory; +import com.ingestpipeline.controller.RestApiController; +import com.ingestpipeline.util.ConfigLoader; +import com.ingestpipeline.util.ReadUtil; + +/** + * The App Startup Runner runs on the start of the application as it implements Application Runner + * This will be responsible to load the configurations which are necessary for the Enrichment in the Data Pipeline + * Resources and Domain level configurations are fetched here. + * @author Pritha + * + */ +@Component +public class AppStartupRunner implements ApplicationRunner { + + private static Logger logger = LoggerFactory.getLogger(AppStartupRunner.class); + + @Autowired + ConfigLoader configLoader; + @Autowired + DomainConfigFactory domainConfigFactory; + + @Autowired ReadUtil readutil; + + @Autowired RestApiController restApiController; + + @Autowired IngestServiceImpl ingestService; + + @SuppressWarnings("static-access") + @Override + public void run(ApplicationArguments args) throws Exception { + logger.info("On Boot starts loading: config resources "); + configLoader.loadResources(); + for(DomainConfig domainConfig : domainConfigFactory.getAllConfigs()) { + domainConfig.loadDomains(); + } + + ingestService.loadTopicsConfig(); + + } +} diff --git a/dashboard-ingest/src/main/java/com/ingestpipeline/IngestApp.java b/dashboard-ingest/src/main/java/com/ingestpipeline/IngestApp.java new file mode 100644 index 000000000..8f5f90b19 --- /dev/null +++ b/dashboard-ingest/src/main/java/com/ingestpipeline/IngestApp.java @@ -0,0 +1,37 @@ +package com.ingestpipeline; + +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.context.annotation.Bean; +import org.springframework.web.client.RestTemplate; +import org.springframework.web.servlet.config.annotation.CorsRegistry; +import org.springframework.web.servlet.config.annotation.WebMvcConfigurer; +import org.springframework.web.servlet.config.annotation.WebMvcConfigurerAdapter; + +import com.ingestpipeline.util.Constants; + + +@SpringBootApplication(scanBasePackages={"com.ingestpipeline"})// same as @Configuration @EnableAutoConfiguration @ComponentScan combined +public class IngestApp { + + public static void main(String[] args) { + SpringApplication.run(IngestApp.class, args); + } + + @Bean + public RestTemplate restTemplate() { + return new RestTemplate(); + } + + @Bean + public WebMvcConfigurer corsConfigurer() { + return new WebMvcConfigurerAdapter() { + @Override + public void addCorsMappings(CorsRegistry registry) { + registry.addMapping("/**").allowedMethods(Constants.ALLOWED_METHODS_GET,Constants.ALLOWED_METHODS_POST + ).allowedOrigins("*") + .allowedHeaders("*"); + } + }; + } +} diff --git a/dashboard-ingest/src/main/java/com/ingestpipeline/config/DomainConfig.java b/dashboard-ingest/src/main/java/com/ingestpipeline/config/DomainConfig.java new file mode 100644 index 000000000..5e553117b --- /dev/null +++ b/dashboard-ingest/src/main/java/com/ingestpipeline/config/DomainConfig.java @@ -0,0 +1,11 @@ +package com.ingestpipeline.config; + +import com.ingestpipeline.model.DomainIndexConfig; + +public interface DomainConfig { + + public static final String DOMAIN_CONFIG ="domainConfig"; + public DomainIndexConfig getIndexConfig(String domainName); + public void loadDomains(); + +} diff --git a/dashboard-ingest/src/main/java/com/ingestpipeline/config/DomainConfigFactory.java b/dashboard-ingest/src/main/java/com/ingestpipeline/config/DomainConfigFactory.java new file mode 100644 index 000000000..67951c31f --- /dev/null +++ b/dashboard-ingest/src/main/java/com/ingestpipeline/config/DomainConfigFactory.java @@ -0,0 +1,39 @@ +package com.ingestpipeline.config; + +import java.util.ArrayList; +import java.util.List; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +import com.ingestpipeline.configfactory.CollectionDomainConfig; +import com.ingestpipeline.configfactory.EnhanceDomainConfig; + +@Component +public class DomainConfigFactory { + + @Autowired + private CollectionDomainConfig collectionDomainConfig; + + @Autowired + private EnhanceDomainConfig enhanceDomainConfig; + + public DomainConfig getConfiguration(String type) { + + if (type.equals("collection")) { + return collectionDomainConfig; + + } else if(type.equals("dataEnhancement")) { + return enhanceDomainConfig; + + } + return null; + } + + public List getAllConfigs() { + List domainConfigList = new ArrayList<>(); + domainConfigList.add(collectionDomainConfig); + domainConfigList.add(enhanceDomainConfig); + return domainConfigList; + } +} diff --git a/dashboard-ingest/src/main/java/com/ingestpipeline/config/ElasticSearchConfiguration.java b/dashboard-ingest/src/main/java/com/ingestpipeline/config/ElasticSearchConfiguration.java new file mode 100644 index 000000000..2bf420533 --- /dev/null +++ b/dashboard-ingest/src/main/java/com/ingestpipeline/config/ElasticSearchConfiguration.java @@ -0,0 +1,66 @@ +package com.ingestpipeline.config; + +import org.apache.http.HttpHost; +import org.elasticsearch.client.RestClient; +import org.elasticsearch.client.RestHighLevelClient; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.beans.factory.config.AbstractFactoryBean; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +import com.ingestpipeline.service.ElasticService; +import com.ingestpipeline.service.IESService; + +@Configuration +public class ElasticSearchConfiguration extends AbstractFactoryBean { + + private static final Logger logger = LoggerFactory.getLogger(ElasticSearchConfiguration.class); + @Value("${spring.data.elasticsearch.cluster.nodes}") + private String clusterNodes; + @Value("${spring.data.elasticsearch.cluster.name}") + private String clusterName; + private RestHighLevelClient restHighLevelClient; + @Value("${services.esindexer.host.name}") + private String esIndexerHostName; + @Value("${services.esindexer.host.port}") + private int esIndexerHostPort; + + @Override + public void destroy() { + try { + if (restHighLevelClient != null) { + restHighLevelClient.close(); + } + } catch (final Exception e) { + logger.error("Error closing ElasticSearch client: ", e); + } + } + + @Override + public Class getObjectType() { + return RestHighLevelClient.class; + } + + @Override + public boolean isSingleton() { + return false; + } + + @Override + public RestHighLevelClient createInstance() { + return buildClient(); + } + + private RestHighLevelClient buildClient() { + try { + restHighLevelClient = new RestHighLevelClient( + RestClient.builder(new HttpHost(esIndexerHostName, esIndexerHostPort, "http"))); + } catch (Exception e) { + logger.error(e.getMessage()); + } + return restHighLevelClient; + } + +} diff --git a/dashboard-ingest/src/main/java/com/ingestpipeline/configfactory/CollectionDomainConfig.java b/dashboard-ingest/src/main/java/com/ingestpipeline/configfactory/CollectionDomainConfig.java new file mode 100644 index 000000000..070cc71e9 --- /dev/null +++ b/dashboard-ingest/src/main/java/com/ingestpipeline/configfactory/CollectionDomainConfig.java @@ -0,0 +1,74 @@ +package com.ingestpipeline.configfactory; + +import java.util.HashMap; +import java.util.Iterator; +import java.util.Map; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.ingestpipeline.config.DomainConfig; +import com.ingestpipeline.model.DomainIndexConfig; +import com.ingestpipeline.util.ConfigLoader; +import com.ingestpipeline.util.Constants; + +//@JsonIgnoreProperties(ignoreUnknown=true) +@Component(Constants.DomainConfigurations.COLLECTION_DOMAIN_CONFIG) +public class CollectionDomainConfig implements DomainConfig { + + private static final Logger LOGGER = LoggerFactory.getLogger(CollectionDomainConfig.class); + private static final String COLLECTION_DOMAIN_CONFIG = Constants.DomainConfigurations.COLLECTION_DOMAIN_CONFIG + ".json"; + + + @Autowired + private ConfigLoader configLoader; + /** + * Holds domain name as key and it's index config detail as value. + */ + private Map domainIndexConfigMap = new HashMap<>(); + + public void putDomain(String domainName, DomainIndexConfig domainIndexConfig){ + domainIndexConfigMap.put(domainName, domainIndexConfig); + } + + @Override + public DomainIndexConfig getIndexConfig(String domainName){ + return domainIndexConfigMap.get(domainName); + } + + + /** + * loads once on application start up. + */ + @Override + public void loadDomains(){ + String collectionConfigContent = configLoader.get(COLLECTION_DOMAIN_CONFIG); + //LOGGER.info("collectionConfigContent json string = "+collectionConfigContent); + + try{ + + ObjectMapper mapper = new ObjectMapper(); + JsonNode root = new ObjectMapper().readTree(collectionConfigContent); + ArrayNode domainConfigArr = (ArrayNode) root.path(DOMAIN_CONFIG); + + Iterator iterator = domainConfigArr.elements(); + while (iterator.hasNext()) { + DomainIndexConfig domainIndexConfig = mapper.readValue(iterator.next().toString(), DomainIndexConfig.class); + LOGGER.info("DomainIndexConfig id:: " + domainIndexConfig.getDomain()); + domainIndexConfigMap.put(domainIndexConfig.getDomain(), domainIndexConfig); + + } + LOGGER.info("After loading, domainIndexConfigMap size = "+ domainIndexConfigMap.size()); + + } catch (Exception e){ + e.printStackTrace(); + LOGGER.error("on construction domain collection map: "+ e.getMessage()); + } + + } +} diff --git a/dashboard-ingest/src/main/java/com/ingestpipeline/configfactory/EnhanceDomainConfig.java b/dashboard-ingest/src/main/java/com/ingestpipeline/configfactory/EnhanceDomainConfig.java new file mode 100644 index 000000000..0dea3c62a --- /dev/null +++ b/dashboard-ingest/src/main/java/com/ingestpipeline/configfactory/EnhanceDomainConfig.java @@ -0,0 +1,75 @@ +package com.ingestpipeline.configfactory; + +import java.util.HashMap; +import java.util.Iterator; +import java.util.Map; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.ingestpipeline.config.DomainConfig; +import com.ingestpipeline.model.DomainIndexConfig; +import com.ingestpipeline.util.ConfigLoader; +import com.ingestpipeline.util.Constants; + +//@JsonIgnoreProperties(ignoreUnknown=true) +@Component(Constants.DomainConfigurations.ENHANCE_DOMAIN_CONFIG) +public class EnhanceDomainConfig implements DomainConfig { + + private static final Logger LOGGER = LoggerFactory.getLogger(EnhanceDomainConfig.class); + private static final String ENHANCE_DOMAIN_CONFIG = Constants.DomainConfigurations.ENHANCE_DOMAIN_CONFIG + ".json"; + + + @Autowired + private ConfigLoader configLoader; + /** + * Holds domain name as key and it's index config detail as value. + */ + private Map domainIndexConfigMap = new HashMap<>(); + + public void putDomain(String domainName, DomainIndexConfig domainIndexConfig){ + domainIndexConfigMap.put(domainName, domainIndexConfig); + } + + @Override + public DomainIndexConfig getIndexConfig(String domainName){ + return domainIndexConfigMap.get(domainName); + } + + + /** + * loads once on application start up. + */ + @Override + public void loadDomains(){ + String enhanceConfigContent = configLoader.get(ENHANCE_DOMAIN_CONFIG); + // LOGGER.info("collectionConfigContent json string = "+collectionConfigContent); + LOGGER.info("collectionConfigContent json string = "+enhanceConfigContent); + + try{ + + ObjectMapper mapper = new ObjectMapper(); + JsonNode root = new ObjectMapper().readTree(enhanceConfigContent); + ArrayNode domainConfigArr = (ArrayNode) root.path(DOMAIN_CONFIG); + + Iterator iterator = domainConfigArr.elements(); + while (iterator.hasNext()) { + DomainIndexConfig domainIndexConfig = mapper.readValue(iterator.next().toString(), DomainIndexConfig.class); + LOGGER.info("DomainIndexConfig id:: " + domainIndexConfig.getDomain()); + domainIndexConfigMap.put(domainIndexConfig.getDomain(), domainIndexConfig); + + } + LOGGER.info("After loading, domainIndexConfigMap size = "+ domainIndexConfigMap.size()); + + } catch (Exception e){ + e.printStackTrace(); + LOGGER.error("on construction domain collection map: "+ e.getMessage()); + } + + } +} diff --git a/dashboard-ingest/src/main/java/com/ingestpipeline/consumer/ConsumerConfigurations.java b/dashboard-ingest/src/main/java/com/ingestpipeline/consumer/ConsumerConfigurations.java new file mode 100644 index 000000000..078db23f3 --- /dev/null +++ b/dashboard-ingest/src/main/java/com/ingestpipeline/consumer/ConsumerConfigurations.java @@ -0,0 +1,127 @@ +package com.ingestpipeline.consumer; + +import java.util.HashMap; +import java.util.Map; + +import org.apache.kafka.clients.consumer.ConsumerConfig; +import org.apache.kafka.common.serialization.StringDeserializer; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.PropertySource; +import org.springframework.kafka.annotation.EnableKafka; +import org.springframework.kafka.config.ConcurrentKafkaListenerContainerFactory; +import org.springframework.kafka.core.ConsumerFactory; +import org.springframework.kafka.core.DefaultKafkaConsumerFactory; +import org.springframework.kafka.support.serializer.JsonDeserializer; + +/** + * The configurations for the Kafka Consumers are fed from this Config Class + * @author Darshan Nagesh + * + */ +@PropertySource(value= {"classpath:application.properties"}) +@Configuration +@EnableKafka +public class ConsumerConfigurations { + + @Autowired + org.springframework.core.env.Environment env; + + @Value("${spring.kafka.bootstrap.servers}") + private String serverConfig; + + @Value("${kafka.consumer.config.auto_commit}") + private Boolean enableAutoCommit; + + @Value("${kafka.consumer.config.auto_commit_interval}") + private String autoCommitInterval; + + @Value("${kafka.consumer.config.session_timeout}") + private String sessionTimeout; + + @Value("${kafka.consumer.config.group_id}") + private String groupId; + + @Value("${kafka.consumer.config.auto_offset_reset}") + private String autoOffsetReset; + + @Value("${spring.kafka.consumer.value-deserializer}") + private String valueDeserializer; + + @Value("${spring.kafka.consumer.key-deserializer}") + private String keyDeserializer; + + + public ConsumerFactory kafkaConsumerFactory() { + JsonDeserializer deserializer = new JsonDeserializer<>(Map.class); + //deserializer.setRemoveTypeHeaders(false); + deserializer.addTrustedPackages("*"); + deserializer.setUseTypeMapperForKey(true); + + Map props = new HashMap<>(); + props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, serverConfig); + props.put(ConsumerConfig.GROUP_ID_CONFIG, groupId); + return new DefaultKafkaConsumerFactory<>(props, new StringDeserializer(), deserializer); + } + + @Bean + public ConcurrentKafkaListenerContainerFactory incomingKafkaListenerContainerFactory() { + ConcurrentKafkaListenerContainerFactory factory = new ConcurrentKafkaListenerContainerFactory<>(); + factory.setConsumerFactory(kafkaConsumerFactory()); + return factory; + } + + + + /*@Bean + KafkaListenerContainerFactory> kafkaListenerContainerFactory() { + System.out.println("kafkaListenerContainerFactory"); + final ConcurrentKafkaListenerContainerFactory factory = new ConcurrentKafkaListenerContainerFactory<>(); + factory.setConsumerFactory(consumerFactory()); + factory.setConcurrency(3); + factory.getContainerProperties().setPollTimeout(3000); + return factory; + } + + @Bean + public ConsumerFactory consumerFactory() { + System.out.println("consumerFactory"); + return new DefaultKafkaConsumerFactory<>(consumerConfigs()); + } + + @Bean + public Map consumerConfigs() { + // TODO - Load configs from env vars + final Map propsMap = new HashMap<>(); + propsMap.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, serverConfig); + propsMap.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, enableAutoCommit); + propsMap.put(ConsumerConfig.AUTO_COMMIT_INTERVAL_MS_CONFIG, autoCommitInterval); + propsMap.put(ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG, sessionTimeout); + propsMap.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, autoOffsetReset); + propsMap.put(ConsumerConfig.GROUP_ID_CONFIG, groupId); + propsMap.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class); + propsMap.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, HashMapDeserializer.class); +*//* propsMap.put(JsonDeserializer., "*"); +*//* return propsMap; + } + + @Bean + public ConcurrentKafkaListenerContainerFactory incomingKafkaListenerContainerFactory() { + ConcurrentKafkaListenerContainerFactory factory = new ConcurrentKafkaListenerContainerFactory<>(); + factory.setConsumerFactory(taskConsumerFactory()); + return factory; + } + + public ConsumerFactory taskConsumerFactory() { + Map props = new HashMap<>(); + props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9092"); + props.put(ConsumerConfig.GROUP_ID_CONFIG, "task-group-notify"); + return new DefaultKafkaConsumerFactory<>(props, new StringDeserializer(), new JsonDeserializer<>(IncomingData.class)); + } +*/ + /* + * @Bean public TransactionPersistConsumer listener() { return new TransactionPersistConsumer(); } + */ +} \ No newline at end of file diff --git a/dashboard-ingest/src/main/java/com/ingestpipeline/consumer/DigressionConsumer.java b/dashboard-ingest/src/main/java/com/ingestpipeline/consumer/DigressionConsumer.java new file mode 100644 index 000000000..ebca3f16c --- /dev/null +++ b/dashboard-ingest/src/main/java/com/ingestpipeline/consumer/DigressionConsumer.java @@ -0,0 +1,34 @@ +package com.ingestpipeline.consumer; + +import java.util.Map; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.kafka.annotation.KafkaListener; +import org.springframework.kafka.support.KafkaHeaders; +import org.springframework.messaging.handler.annotation.Header; + +import com.ingestpipeline.service.DigressService; +import com.ingestpipeline.util.Constants; + +public class DigressionConsumer implements KafkaConsumer { + public static final Logger LOGGER = LoggerFactory.getLogger(DigressionConsumer.class); + public static final String INTENT = "digress"; + + @Autowired + private DigressService digressService; + + @Override + @KafkaListener(topics = { Constants.KafkaTopics.VALID_DATA }, containerFactory = Constants.BeanContainerFactory.INCOMING_KAFKA_LISTENER) + public void processMessage(Map incomingData, + @Header(KafkaHeaders.RECEIVED_TOPIC) final String topic) { + LOGGER.info("##KafkaMessageAlert## : key:" + topic + ":" + "value:" + incomingData.size()); + try { + + } catch (final Exception e) { + LOGGER.error("Exception Encountered while processing the received message : " + e.getMessage()); + } + } + +} diff --git a/dashboard-ingest/src/main/java/com/ingestpipeline/consumer/EnrichmentConsumer.java b/dashboard-ingest/src/main/java/com/ingestpipeline/consumer/EnrichmentConsumer.java new file mode 100644 index 000000000..92f8c44a2 --- /dev/null +++ b/dashboard-ingest/src/main/java/com/ingestpipeline/consumer/EnrichmentConsumer.java @@ -0,0 +1,84 @@ +package com.ingestpipeline.consumer; + +import java.util.Map; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.kafka.annotation.KafkaListener; +import org.springframework.kafka.support.KafkaHeaders; +import org.springframework.messaging.handler.annotation.Header; +import org.springframework.stereotype.Service; + +import com.ingestpipeline.producer.IngestProducer; +import com.ingestpipeline.service.EnrichmentService; +import com.ingestpipeline.service.IESService; +import com.ingestpipeline.util.Constants; + +@Service +public class EnrichmentConsumer implements KafkaConsumer { + + public static final Logger LOGGER = LoggerFactory.getLogger(EnrichmentConsumer.class); + private static final String ERROR_INTENT = "DataError"; + private static final String INTENT = "enrichment" ; + + @Value("${es.push.direct}") + private Boolean esPushDirect; + + @Value("${kafka.topics.enriched.data}") + private String enrichedDataTopic; + + @Value("${kafka.topics.enriched.key}") + private String enrichedDataKey; + + @Autowired + private EnrichmentService enrichmentService; + + @Autowired + private IngestProducer ingestProducer; + + @Autowired + private IESService elasticService; + + @KafkaListener(id = INTENT, groupId = INTENT, topics = { Constants.KafkaTopics.TRANSFORMED_DATA}, containerFactory = Constants.BeanContainerFactory.INCOMING_KAFKA_LISTENER) + public void processMessage(final Map incomingData, + @Header(KafkaHeaders.RECEIVED_TOPIC) final String topic) { + + LOGGER.info("##KafkaMessageAlert## : key:" + topic + ":" + "value:" + incomingData.size()); + + if(incomingData.get(Constants.DATA_CONTEXT) != null && !incomingData.get(Constants.DATA_CONTEXT).equals(Constants.TransformationType.COLLECTION) ){ + push(incomingData, null); + } else { + for(Object key : incomingData.keySet()){ + Map data=(Map)incomingData.get(key); + String trxid = (String)((Map)data.get(Constants.DATA_OBJECT)).get(Constants.ID); + push((Map)incomingData.get(key), trxid); + } + } + } + + private void push(Map incomingData, String docId ) { + + try { + Map updatedIncomingData = enrichmentService.enrichData(incomingData); + + if(esPushDirect) { + LOGGER.info("##esPushDirect## : "+esPushDirect); + elasticService.push(incomingData); + } else { + LOGGER.info("Pushing to:: {} :: docId:: {}" , enrichedDataTopic, " docId:: "+docId); + ingestProducer.pushToPipeline(incomingData, enrichedDataTopic, docId); + } + if(updatedIncomingData == null) { + LOGGER.info("Incoming Data is null::"); + ingestProducer.pushToPipeline(incomingData, ERROR_INTENT, ERROR_INTENT); + } + } catch (final Exception e) { + e.printStackTrace(); + LOGGER.error("Exception Encountered while processing the received message : " + e.getMessage()); + } + + } + +} diff --git a/dashboard-ingest/src/main/java/com/ingestpipeline/consumer/HashMapDeserializer.java b/dashboard-ingest/src/main/java/com/ingestpipeline/consumer/HashMapDeserializer.java new file mode 100644 index 000000000..76941d4f3 --- /dev/null +++ b/dashboard-ingest/src/main/java/com/ingestpipeline/consumer/HashMapDeserializer.java @@ -0,0 +1,14 @@ +package com.ingestpipeline.consumer; + +import java.util.HashMap; + +import org.springframework.kafka.support.serializer.JsonDeserializer; + +@SuppressWarnings("rawtypes") +public class HashMapDeserializer extends JsonDeserializer { + + public HashMapDeserializer() { + super(HashMap.class); + } + +} \ No newline at end of file diff --git a/dashboard-ingest/src/main/java/com/ingestpipeline/consumer/IngestConsumer.java b/dashboard-ingest/src/main/java/com/ingestpipeline/consumer/IngestConsumer.java new file mode 100644 index 000000000..65a3b7441 --- /dev/null +++ b/dashboard-ingest/src/main/java/com/ingestpipeline/consumer/IngestConsumer.java @@ -0,0 +1,45 @@ +package com.ingestpipeline.consumer; + +import java.util.Map; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.kafka.annotation.KafkaListener; +import org.springframework.kafka.support.KafkaHeaders; +import org.springframework.messaging.handler.annotation.Header; +import org.springframework.stereotype.Service; + +import com.ingestpipeline.model.IncomingData; +import com.ingestpipeline.service.IngestService; +import com.ingestpipeline.util.Constants; + +@Service +public class IngestConsumer{ + public static final Logger LOGGER = LoggerFactory.getLogger(IngestConsumer.class); + + @Autowired + private IngestService ingestService; + + @KafkaListener(topics = { "${kafka.topics.incoming.collection}" }, containerFactory = Constants.BeanContainerFactory.INCOMING_KAFKA_LISTENER) + public void processMessage(Map data, + @Header(KafkaHeaders.RECEIVED_TOPIC) final String topic) { + LOGGER.info("##KafkaMessageAlert## : key:" + topic + ":" + "value:" + data.size()); + try { + LOGGER.info("IngestConsumer ## get data ## " +data +" ## TOPIC ## "+topic); + IncomingData incomingData = ingestService.getContextForIncomingTopic(topic); + + LOGGER.info("## incomingData at IngestConsumer "+incomingData); + incomingData.setDataObject(data.get("Data")); + //incomingData.setDataObject(data); + LOGGER.info("## incomingData at IngestConsumer "+incomingData); + + ingestService.ingestToPipeline(incomingData); + } catch (final Exception e) { + LOGGER.error("Exception Encountered while processing the received message : " + e.getMessage()); + e.printStackTrace(); + } + } + + +} diff --git a/dashboard-ingest/src/main/java/com/ingestpipeline/consumer/KafkaConsumer.java b/dashboard-ingest/src/main/java/com/ingestpipeline/consumer/KafkaConsumer.java new file mode 100644 index 000000000..69374842c --- /dev/null +++ b/dashboard-ingest/src/main/java/com/ingestpipeline/consumer/KafkaConsumer.java @@ -0,0 +1,15 @@ +package com.ingestpipeline.consumer; + +import org.springframework.kafka.support.KafkaHeaders; +import org.springframework.messaging.handler.annotation.Header; + +import com.ingestpipeline.model.IncomingData; + +import java.util.Map; + +public interface KafkaConsumer { + + public void processMessage(final Map incomingData, + @Header(KafkaHeaders.RECEIVED_TOPIC) final String topic); + +} diff --git a/dashboard-ingest/src/main/java/com/ingestpipeline/consumer/TransformConsumer.java b/dashboard-ingest/src/main/java/com/ingestpipeline/consumer/TransformConsumer.java new file mode 100644 index 000000000..c5d320a7f --- /dev/null +++ b/dashboard-ingest/src/main/java/com/ingestpipeline/consumer/TransformConsumer.java @@ -0,0 +1,65 @@ +package com.ingestpipeline.consumer; + +import java.util.Map; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.kafka.annotation.KafkaListener; +import org.springframework.kafka.support.KafkaHeaders; +import org.springframework.messaging.handler.annotation.Header; +import org.springframework.stereotype.Service; + +import com.ingestpipeline.producer.IngestProducer; +import com.ingestpipeline.service.TransformService; +import com.ingestpipeline.util.ApplicationProperties; +import com.ingestpipeline.util.Constants; + +@Service +public class TransformConsumer implements KafkaConsumer { + + public static final Logger LOGGER = LoggerFactory.getLogger(TransformConsumer.class); + public static final String INTENT = "transform"; + + @Autowired + @Qualifier(Constants.Qualifiers.TRANSFORM_COLLECTION_SERVICE) + private TransformService collectiontransformService; + + @Autowired + @Qualifier(Constants.Qualifiers.TRANSFORM_SERVICE) + private TransformService defaulttransformService; + + @Autowired + private IngestProducer ingestProducer; + + @Autowired + private ApplicationProperties applicationProperties; + + @Override + @KafkaListener(id = INTENT, groupId = INTENT, topics = { Constants.KafkaTopics.VALID_DATA }, containerFactory = Constants.BeanContainerFactory.INCOMING_KAFKA_LISTENER) + public void processMessage(Map incomingData, + @Header(KafkaHeaders.RECEIVED_TOPIC) final String topic) { + LOGGER.info("##KafkaMessageAlert## : key:" + topic + ":" + "value:" + incomingData.size()); + try { + boolean isTransformed = false; + String dataContext = incomingData.get(Constants.DATA_CONTEXT).toString(); + LOGGER.info("incomingData at TransformConsumer=="+incomingData.toString()); + if(dataContext.equals(Constants.TransformationType.COLLECTION)) { + + isTransformed = collectiontransformService.transformData(incomingData); + } else { + + isTransformed = defaulttransformService.transformData(incomingData); + } + if (isTransformed) { + ingestProducer.pushToPipeline(incomingData, applicationProperties.getTransactionTransformationTopic(), applicationProperties.getTransactionTransformationKey()); + } /*else { + ingestProducer.pushToPipeline(incomingData, Constants.KafkaTopics.ERROR_INTENT, Constants.KafkaTopics.ERROR_INTENT); + }*/ + } catch (final Exception e) { + LOGGER.error("Exception Encountered while processing the received message : " + e.getMessage()); + e.printStackTrace(); + } + } +} diff --git a/dashboard-ingest/src/main/java/com/ingestpipeline/consumer/UpdateConsumer.java b/dashboard-ingest/src/main/java/com/ingestpipeline/consumer/UpdateConsumer.java new file mode 100644 index 000000000..8e422d0b4 --- /dev/null +++ b/dashboard-ingest/src/main/java/com/ingestpipeline/consumer/UpdateConsumer.java @@ -0,0 +1,69 @@ +package com.ingestpipeline.consumer; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.ingestpipeline.producer.IngestProducer; +import com.ingestpipeline.service.IESService; +import com.ingestpipeline.util.Constants; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.http.ResponseEntity; +import org.springframework.kafka.annotation.KafkaListener; +import org.springframework.kafka.support.KafkaHeaders; +import org.springframework.messaging.handler.annotation.Header; +import org.springframework.stereotype.Component; + +import java.util.Map; +import java.net.URLEncoder; + +@Component +public class UpdateConsumer { + + + public static final Logger LOGGER = LoggerFactory.getLogger(UpdateConsumer.class); + private static final String ERROR_INTENT = "DataErrorBypass"; + @Autowired + private IngestProducer ingestProducer; + @Autowired + private ObjectMapper mapper; + @Autowired + private IESService elasticService; + + @Value("${kafka.topics.bypass.update.post}") + String updatedPostTopic; + + @Value("${es.bypass.push.direct}") + private Boolean esPushDirect; + + @KafkaListener(topics = "${kafka.topics.bypass.update.data}" , containerFactory = Constants.BeanContainerFactory.INCOMING_KAFKA_LISTENER) + public void processMessage(Map data, + @Header(KafkaHeaders.RECEIVED_TOPIC) final String topic) { + LOGGER.info("##KafkaMessageAlert## : key:" + topic + ":" + "value:" + data.size()); + LOGGER.info("data at UpdateConsumer: :"+data); + try { + + String index = data.get("_index").toString(); + String type = data.get("_type").toString(); + JsonNode sourceNode = mapper.convertValue(data.get("_source"), JsonNode.class); + String _id = data.get("_id").toString(); + String id = URLEncoder.encode(data.get("_id").toString()); + + if(esPushDirect){ + ResponseEntity response = elasticService.post(index, type, id, "", sourceNode.toString()); + LOGGER.info("index :: {}, Response :: {} " ,index , response.getStatusCode()); + } else { + ingestProducer.pushToPipeline(sourceNode, updatedPostTopic, _id); + } + + } catch (final Exception e) { + e.printStackTrace(); + if(!esPushDirect) + ingestProducer.pushToPipeline(data, ERROR_INTENT, ERROR_INTENT); + LOGGER.error("Exception Encountered while processing the received message updating posted data for topic: "+ topic +"" + e.getMessage()); + } + } + + +} diff --git a/dashboard-ingest/src/main/java/com/ingestpipeline/consumer/ValidatorConsumer.java b/dashboard-ingest/src/main/java/com/ingestpipeline/consumer/ValidatorConsumer.java new file mode 100644 index 000000000..94a4deb9d --- /dev/null +++ b/dashboard-ingest/src/main/java/com/ingestpipeline/consumer/ValidatorConsumer.java @@ -0,0 +1,65 @@ +package com.ingestpipeline.consumer; + +import java.util.Map; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.kafka.annotation.KafkaListener; +import org.springframework.kafka.support.KafkaHeaders; +import org.springframework.messaging.handler.annotation.Header; +import org.springframework.stereotype.Service; + +import com.ingestpipeline.producer.IngestProducer; +import com.ingestpipeline.service.ValidationService; +import com.ingestpipeline.util.ApplicationProperties; +import com.ingestpipeline.util.Constants; + +@Service +public class ValidatorConsumer implements KafkaConsumer { + + public static final Logger LOGGER = LoggerFactory.getLogger(ValidatorConsumer.class); + + public static final String INTENT = "validator"; + + @Autowired + private ValidationService validationService; + + @Autowired + private IngestProducer ingestProducer; + + @Autowired + private ApplicationProperties applicationProperties; + + @KafkaListener(id = INTENT, groupId = INTENT, topics = {Constants.KafkaTopics.INGEST_DATA} , containerFactory = Constants.BeanContainerFactory.INCOMING_KAFKA_LISTENER) + public void processMessage(Map consumerRecord, + @Header(KafkaHeaders.RECEIVED_TOPIC) final String topic) { + LOGGER.info("##KafkaMessageAlert## Message Received at Validator Consumer : key:" + topic + ":" + "value:" + consumerRecord.size()); + try { + boolean isValid = validationService.validateData(consumerRecord); + String nextTopic = ""; + String nextKey = ""; + if (isValid) { + if(applicationProperties.getPipelineRules().get(Constants.PipelineRules.TRANSFORM_DATA)) { + nextTopic = applicationProperties.getTransactionValidationTopic(); + nextKey = applicationProperties.getTransactionValidationKey(); + } else if(applicationProperties.getPipelineRules().get(Constants.PipelineRules.ENRICH_DATA)) { + nextTopic = applicationProperties.getTransactionTransformationTopic(); + nextKey = applicationProperties.getTransactionTransformationKey(); + } + ingestProducer.pushToPipeline(consumerRecord, nextTopic, nextKey); + } else { + ingestProducer.pushToPipeline(consumerRecord, Constants.KafkaTopics.ERROR_INTENT, Constants.KafkaTopics.ERROR_INTENT); + } + LOGGER.info("Next Topic: " + nextTopic); + LOGGER.info("Next Key: " + nextKey); + + } catch (final Exception e) { + e.printStackTrace(); + LOGGER.error("Exception Encountered while processing the received message : " + e.getMessage()); + } + + + } + +} diff --git a/dashboard-ingest/src/main/java/com/ingestpipeline/controller/ProducerController.java b/dashboard-ingest/src/main/java/com/ingestpipeline/controller/ProducerController.java new file mode 100644 index 000000000..8525525e1 --- /dev/null +++ b/dashboard-ingest/src/main/java/com/ingestpipeline/controller/ProducerController.java @@ -0,0 +1,74 @@ +package com.ingestpipeline.controller; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.ingestpipeline.model.Response; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.kafka.core.KafkaTemplate; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RestController; + + +@RestController +public class ProducerController { + + public static final Logger LOGGER = LoggerFactory.getLogger(ProducerController.class); + + @Value("${kafka.topics.bypass.update.data}") + private String topic; + @Value("${kafka.topics.bypass.update.key}") + private String topickey; + @Autowired + private KafkaTemplate kafkaTemplate; + /* + @PostMapping("/update/publish") + public ResponseEntity publish(@RequestBody String body){ + LOGGER.info("publishing request body "+body); + try{ + + JsonNode bodyNode = new ObjectMapper().readTree(body); + ArrayNode nodes = (ArrayNode) bodyNode.get("data"); + LOGGER.info("## nodes ## "+nodes); + for(JsonNode node : nodes ){ + LOGGER.info("single node "+node); + kafkaTemplate.send(topic, topickey, node); + + } + LOGGER.info("Published successfully"); + return new ResponseEntity(new Response("sucessful", "Published successfully"), HttpStatus.OK); + + } catch (Exception e){ + LOGGER.error("Published failed "+ e.getMessage()); + return new ResponseEntity(new Response("failed", e.getMessage()), HttpStatus.BAD_REQUEST); + } + + } +*/ + @Value("${kafka.topics.incoming.collection}") + private String testTopic; + @Value("${kafka.topics.enriched.key}") + private String testKey; + + @PostMapping("/update/collection/test") + public String test(@RequestBody String body){ + try{ + JsonNode node = new ObjectMapper().readTree(body); + kafkaTemplate.send(testTopic, testKey, node.get("data")); + LOGGER.info("Published successfully"); + return "Published successfully"; + + } catch (Exception e){ + LOGGER.error("Published failed "+ e.getMessage()); + return "Published failed: "+e.getMessage(); + } + + } + +} diff --git a/dashboard-ingest/src/main/java/com/ingestpipeline/controller/RestApiController.java b/dashboard-ingest/src/main/java/com/ingestpipeline/controller/RestApiController.java new file mode 100644 index 000000000..67d7f5c43 --- /dev/null +++ b/dashboard-ingest/src/main/java/com/ingestpipeline/controller/RestApiController.java @@ -0,0 +1,189 @@ +package com.ingestpipeline.controller; + +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Date; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.TimeZone; + +import org.json.JSONArray; +import org.json.JSONObject; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.kafka.config.KafkaListenerEndpointRegistry; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; +import org.springframework.web.multipart.MultipartFile; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.gson.Gson; +import com.ingestpipeline.model.IncomingData; +import com.ingestpipeline.model.TargetData; +import com.ingestpipeline.repository.TargetDataDao; +import com.ingestpipeline.service.ElasticService; +import com.ingestpipeline.service.IngestService; +import com.ingestpipeline.util.Constants; +import com.ingestpipeline.util.JSONUtil; +import com.ingestpipeline.util.ReadUtil; + +@RestController +@RequestMapping(Constants.Paths.ELASTIC_PUSH_CONTROLLER_PATH) +public class RestApiController { + + public static final Logger logger = LoggerFactory.getLogger(RestApiController.class); + + @Autowired + IngestService ingestService; + + @Autowired + TargetDataDao targetDataDao; + + @Autowired + private KafkaListenerEndpointRegistry endPointRegistry; + + @Autowired + private ElasticService elasticService; + + /** + * This API use to pause a active kafka consumer + * + * @param consumerId kafka consumer identifier + * @return + */ + @RequestMapping(value = "/pause/{consumerId}", method = RequestMethod.GET) + public Boolean pauseConsumer(@PathVariable final String consumerId) { + endPointRegistry.getListenerContainer(consumerId).pause(); + return Boolean.TRUE; + } + + /** + * This API is to resume a paused kafka consumer + * + * @param consumerId kafka consumer identifier + * @return + */ + @RequestMapping(value = "/resume/{consumerId}", method = RequestMethod.GET) + public Boolean resumeConsumer(@PathVariable final String consumerId) { + endPointRegistry.getListenerContainer(consumerId).resume(); + return Boolean.TRUE; + } + + /** + * This API receives the Transaction Details JSON Request and passes it on to + * the Service Layer for further process of persisting into elastic search + * database + * + * @param transaction + * @return + */ + @RequestMapping(value = Constants.Paths.SAVE, method = RequestMethod.POST) + public ResponseEntity save(@RequestBody IncomingData incomingData) { + logMyTime(); + Boolean status = ingestService.ingestToPipeline(incomingData); + if (status) { + return new ResponseEntity(HttpStatus.CREATED); + } + return new ResponseEntity(HttpStatus.SERVICE_UNAVAILABLE); + } + + /** + * This API use to provide response for external data upload + * + * @param get file + * @author Rahul + * @throws Exception + */ + @RequestMapping(value = Constants.Paths.Targets, method = RequestMethod.GET) + public String getTargets() throws Exception { + List datas = (List) targetDataDao.findAll(); + String response = JSONUtil.getJsonString(new ObjectMapper(), datas); + return response; + } + + /** + * This API use to import external data upload Request + * + * @param upload file + * @author Rahul + * @throws Exception + */ + @RequestMapping(value = Constants.Paths.UPLOAD, method = RequestMethod.POST) + public ResponseEntity handleFileUpload(@RequestParam("file") MultipartFile file) throws Exception { + Boolean status = ingestService.ingestToPipeline(getWrapper(file)); + if (status) { + return new ResponseEntity(HttpStatus.CREATED); + } + return new ResponseEntity(HttpStatus.SERVICE_UNAVAILABLE); + } + + public IncomingData getWrapper(MultipartFile file) throws Exception { + JSONArray jsonArray = new JSONArray(); + jsonArray = ReadUtil.getFiletoDirectory(file); + IncomingData incomingData = new IncomingData(); + incomingData.setDataContext("target"); + incomingData.setDataContextVersion("v1"); + Iterator itr = jsonArray.iterator(); + List list = new ArrayList(); + while (itr.hasNext()) { + JSONObject obj = (JSONObject) itr.next(); + Map mapJ = new Gson().fromJson(obj.toString(), Map.class); + list.add(mapJ); + } + incomingData.setDataObject(list); + return incomingData; + } + + /** + * This API to post documents from ES index + * + * @param post documents + * @author Rahul + * @throws Exception + */ + @RequestMapping(value = "/migrate/{indexName}/{version}", method = RequestMethod.POST) + public ResponseEntity migrateIndex(@PathVariable String indexName, @PathVariable String version) throws Exception { + String index = null, queryString = null, dataContext = null; + Boolean status = elasticService.searchIndex(indexName, queryString, version); + if (status) { + return new ResponseEntity(HttpStatus.CREATED); + } else { + return new ResponseEntity(HttpStatus.NOT_FOUND); + } + } + +/** + * This API to post documents from ES index + * + * @param post documents + * @author Rahul + * @throws Exception + */ + @RequestMapping(value = "/migrateV2/{indexName}/{version}", method = RequestMethod.GET) + public ResponseEntity migrateIndexV2(@PathVariable String indexName, @PathVariable String dataContextVersion) throws Exception { + String index = null, queryString = null, dataContext = null; + Boolean status = elasticService.searchIndex(indexName, queryString, dataContextVersion); + if (status) { + return new ResponseEntity(HttpStatus.CREATED); + } else if (index.equals("notDefinedIndex")) { + return new ResponseEntity(HttpStatus.NOT_FOUND); + } + return new ResponseEntity(HttpStatus.SERVICE_UNAVAILABLE); + } + + private void logMyTime() { + logger.info("System Time is : " + new Date()); + SimpleDateFormat sd = new SimpleDateFormat(Constants.DATE_FORMAT); + Date date = new Date(); + sd.setTimeZone(TimeZone.getTimeZone(Constants.INDIAN_TIMEZONE)); + logger.info("Time at timezone IST : " + sd.format(date)); + } +} \ No newline at end of file diff --git a/dashboard-ingest/src/main/java/com/ingestpipeline/model/Consumable.java b/dashboard-ingest/src/main/java/com/ingestpipeline/model/Consumable.java new file mode 100644 index 000000000..7f34f9f95 --- /dev/null +++ b/dashboard-ingest/src/main/java/com/ingestpipeline/model/Consumable.java @@ -0,0 +1,58 @@ +package com.ingestpipeline.model; + + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import com.fasterxml.jackson.annotation.JsonAnyGetter; +import com.fasterxml.jackson.annotation.JsonAnySetter; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonPropertyOrder; + +@JsonInclude(JsonInclude.Include.NON_NULL) +@JsonPropertyOrder({ +"dataContext", +"topics" +}) +public class Consumable { + +@JsonProperty("dataContext") +private String dataContext; +@JsonProperty("topics") +private List topics = null; +@JsonIgnore +private Map additionalProperties = new HashMap(); + +@JsonProperty("dataContext") +public String getDataContext() { +return dataContext; +} + +@JsonProperty("dataContext") +public void setDataContext(String dataContext) { +this.dataContext = dataContext; +} + +@JsonProperty("topics") +public List getTopics() { +return topics; +} + +@JsonProperty("topics") +public void setTopics(List topics) { +this.topics = topics; +} + +@JsonAnyGetter +public Map getAdditionalProperties() { +return this.additionalProperties; +} + +@JsonAnySetter +public void setAdditionalProperty(String name, Object value) { +this.additionalProperties.put(name, value); +} + +} \ No newline at end of file diff --git a/dashboard-ingest/src/main/java/com/ingestpipeline/model/DigressionPoint.java b/dashboard-ingest/src/main/java/com/ingestpipeline/model/DigressionPoint.java new file mode 100644 index 000000000..e65fd4f7b --- /dev/null +++ b/dashboard-ingest/src/main/java/com/ingestpipeline/model/DigressionPoint.java @@ -0,0 +1,38 @@ +package com.ingestpipeline.model; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import com.fasterxml.jackson.annotation.JsonAnyGetter; +import com.fasterxml.jackson.annotation.JsonAnySetter; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonPropertyOrder; + +@JsonInclude(JsonInclude.Include.NON_NULL) +@JsonPropertyOrder({ "dataContext", "paths" }) +public class DigressionPoint { + + @JsonProperty("dataContext") + private String dataContext; + @JsonProperty("paths") + private List paths = null; + + public String getDataContext() { + return dataContext; + } + + public void setDataContext(String dataContext) { + this.dataContext = dataContext; + } + + public List getPaths() { + return paths; + } + + public void setPaths(List paths) { + this.paths = paths; + } + +} diff --git a/dashboard-ingest/src/main/java/com/ingestpipeline/model/DigressionPointConfig.java b/dashboard-ingest/src/main/java/com/ingestpipeline/model/DigressionPointConfig.java new file mode 100644 index 000000000..94a271e55 --- /dev/null +++ b/dashboard-ingest/src/main/java/com/ingestpipeline/model/DigressionPointConfig.java @@ -0,0 +1,44 @@ +package com.ingestpipeline.model; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import com.fasterxml.jackson.annotation.JsonAnyGetter; +import com.fasterxml.jackson.annotation.JsonAnySetter; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonPropertyOrder; + +@JsonInclude(JsonInclude.Include.NON_NULL) +@JsonPropertyOrder({ +"digressionPoints" +}) +public class DigressionPointConfig { + +@JsonProperty("digressionPoints") +private List digressionPoints = null; +@JsonIgnore +private Map additionalProperties = new HashMap(); + +@JsonProperty("digressionPoints") +public List getDigressionPoints() { +return digressionPoints; +} + +@JsonProperty("digressionPoints") +public void setDigressionPoints(List digressionPoints) { +this.digressionPoints = digressionPoints; +} + +@JsonAnyGetter +public Map getAdditionalProperties() { +return this.additionalProperties; +} + +@JsonAnySetter +public void setAdditionalProperty(String name, Object value) { +this.additionalProperties.put(name, value); +} + +} \ No newline at end of file diff --git a/dashboard-ingest/src/main/java/com/ingestpipeline/model/DomainIndexConfig.java b/dashboard-ingest/src/main/java/com/ingestpipeline/model/DomainIndexConfig.java new file mode 100644 index 000000000..043db28ca --- /dev/null +++ b/dashboard-ingest/src/main/java/com/ingestpipeline/model/DomainIndexConfig.java @@ -0,0 +1,81 @@ +package com.ingestpipeline.model; + +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.util.ArrayList; +import java.util.List; + +public class DomainIndexConfig { + + private String id; + private String domain; + private String indexName; + private String documentType; + private String query; + + private List sourceReferences = new ArrayList<>(); + private List targetReferences = new ArrayList<>(); + + @JsonProperty(value="sourceReferences") + public List getSourceReferences() { + return sourceReferences; + } + + public void setSourceReferences(List sourceReferences) { + this.sourceReferences = sourceReferences; + } + @JsonProperty(value="targetReferences") + public List getTargetReferences() { + return targetReferences; + } + + public void setTargetReferences(List targetReferences) { + this.targetReferences = targetReferences; + } + + @JsonProperty(value="id") + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + @JsonProperty(value="domain") + public String getDomain() { + return domain; + } + + public void setDomain(String businessType) { + this.domain = businessType; + } + + @JsonProperty(value="indexName") + public String getIndexName() { + return indexName; + } + + public void setIndexName(String indexName) { + this.indexName = indexName; + } + + @JsonProperty(value="documentType") + public String getDocumentType() { + return documentType; + } + + public void setDocumentType(String documentType) { + this.documentType = documentType; + } + + @JsonProperty(value="query") + public String getQuery() { + return query; + } + + public void setQuery(String query) { + this.query = query; + } + +} diff --git a/dashboard-ingest/src/main/java/com/ingestpipeline/model/ErrorWrapper.java b/dashboard-ingest/src/main/java/com/ingestpipeline/model/ErrorWrapper.java new file mode 100644 index 000000000..a38b00c64 --- /dev/null +++ b/dashboard-ingest/src/main/java/com/ingestpipeline/model/ErrorWrapper.java @@ -0,0 +1,27 @@ +package com.ingestpipeline.model; + +public class ErrorWrapper { + + private String errorCode; + private String errorMessage; + private Object incomingData; + + public String getErrorCode() { + return errorCode; + } + public void setErrorCode(String errorCode) { + this.errorCode = errorCode; + } + public String getErrorMessage() { + return errorMessage; + } + public void setErrorMessage(String errorMessage) { + this.errorMessage = errorMessage; + } + public Object getIncomingData() { + return incomingData; + } + public void setIncomingData(Object incomingData) { + this.incomingData = incomingData; + } +} diff --git a/dashboard-ingest/src/main/java/com/ingestpipeline/model/IncomingData.java b/dashboard-ingest/src/main/java/com/ingestpipeline/model/IncomingData.java new file mode 100644 index 000000000..ce02ff52f --- /dev/null +++ b/dashboard-ingest/src/main/java/com/ingestpipeline/model/IncomingData.java @@ -0,0 +1,35 @@ +package com.ingestpipeline.model; + +public class IncomingData { + + private String dataContext; + private String dataContextVersion; + private Object dataObject; + + public String getDataContext() { + return dataContext; + } + public void setDataContext(String dataContext) { + this.dataContext = dataContext; + } + public String getDataContextVersion() { + return dataContextVersion; + } + public void setDataContextVersion(String dataContextVersion) { + this.dataContextVersion = dataContextVersion; + } + public Object getDataObject() { + return dataObject; + } + public void setDataObject(Object dataObject) { + this.dataObject = dataObject; + } + @Override + public String toString() { + return "IncomingData [dataContext=" + dataContext + ", dataContextVersion=" + dataContextVersion + + ", dataObject=" + dataObject + "]"; + } + + + +} diff --git a/dashboard-ingest/src/main/java/com/ingestpipeline/model/IncomingDataConfig.java b/dashboard-ingest/src/main/java/com/ingestpipeline/model/IncomingDataConfig.java new file mode 100644 index 000000000..f9ac55dce --- /dev/null +++ b/dashboard-ingest/src/main/java/com/ingestpipeline/model/IncomingDataConfig.java @@ -0,0 +1,45 @@ +package com.ingestpipeline.model; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import com.fasterxml.jackson.annotation.JsonAnyGetter; +import com.fasterxml.jackson.annotation.JsonAnySetter; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonPropertyOrder; + +@JsonInclude(JsonInclude.Include.NON_NULL) +@JsonPropertyOrder({ +"consumables" +}) +public class IncomingDataConfig { + +@JsonProperty("consumables") +private List consumables = null; +@JsonIgnore +private Map additionalProperties = new HashMap(); + +@JsonProperty("consumables") +public List getConsumables() { +return consumables; +} + +@JsonProperty("consumables") +public void setConsumables(List consumables) { +this.consumables = consumables; +} + +@JsonAnyGetter +public Map getAdditionalProperties() { +return this.additionalProperties; +} + +@JsonAnySetter +public void setAdditionalProperty(String name, Object value) { +this.additionalProperties.put(name, value); +} + +} \ No newline at end of file diff --git a/dashboard-ingest/src/main/java/com/ingestpipeline/model/KeyValuePair.java b/dashboard-ingest/src/main/java/com/ingestpipeline/model/KeyValuePair.java new file mode 100644 index 000000000..3556caf66 --- /dev/null +++ b/dashboard-ingest/src/main/java/com/ingestpipeline/model/KeyValuePair.java @@ -0,0 +1,72 @@ +package com.ingestpipeline.model; + +import java.util.HashMap; +import java.util.Map; + +import com.fasterxml.jackson.annotation.JsonAnyGetter; +import com.fasterxml.jackson.annotation.JsonAnySetter; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonPropertyOrder; + +@JsonInclude(JsonInclude.Include.NON_NULL) +@JsonPropertyOrder({ +"name", +"value", +"intent" +}) +public class KeyValuePair { + +@JsonProperty("intent") +private String intent; +@JsonProperty("name") +private String name; +@JsonProperty("value") +private String value; +@JsonIgnore +private Map additionalProperties = new HashMap(); + +@JsonProperty("name") +public String getName() { +return name; +} + +@JsonProperty("name") +public void setName(String name) { +this.name = name; +} + +@JsonProperty("intent") +public String getIntent() { +return intent; +} + +@JsonProperty("intent") +public void setIntent(String intent) { +this.intent = intent; +} + + + +@JsonProperty("value") +public String getValue() { +return value; +} + +@JsonProperty("value") +public void setValue(String value) { +this.value = value; +} + +@JsonAnyGetter +public Map getAdditionalProperties() { +return this.additionalProperties; +} + +@JsonAnySetter +public void setAdditionalProperty(String name, Object value) { +this.additionalProperties.put(name, value); +} + +} diff --git a/dashboard-ingest/src/main/java/com/ingestpipeline/model/Path.java b/dashboard-ingest/src/main/java/com/ingestpipeline/model/Path.java new file mode 100644 index 000000000..38c35ed94 --- /dev/null +++ b/dashboard-ingest/src/main/java/com/ingestpipeline/model/Path.java @@ -0,0 +1,65 @@ +package com.ingestpipeline.model; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import com.fasterxml.jackson.annotation.JsonAnyGetter; +import com.fasterxml.jackson.annotation.JsonAnySetter; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonPropertyOrder; + +@JsonInclude(JsonInclude.Include.NON_NULL) +@JsonPropertyOrder({ "route", "paths" }) +public class Path { + + @JsonProperty("route") + private String route; + + @JsonProperty("paths") + private List paths; + + @JsonProperty("removeReplace") + private String removeReplace; + + public String getRemoveReplace() { + return removeReplace; + } + + public void setRemoveReplace(String removeReplace) { + this.removeReplace = removeReplace; + } + + public List getPaths() { + return paths; + } + + public void setPaths(List paths) { + this.paths = paths; + } + + @JsonIgnore + private Map additionalProperties = new HashMap(); + + @JsonProperty("route") + public String getRoute() { + return route; + } + + @JsonProperty("route") + public void setRoute(String route) { + this.route = route; + } + + @JsonAnyGetter + public Map getAdditionalProperties() { + return this.additionalProperties; + } + + @JsonAnySetter + public void setAdditionalProperty(String name, Object value) { + this.additionalProperties.put(name, value); + } +} \ No newline at end of file diff --git a/dashboard-ingest/src/main/java/com/ingestpipeline/model/References.java b/dashboard-ingest/src/main/java/com/ingestpipeline/model/References.java new file mode 100644 index 000000000..130eac810 --- /dev/null +++ b/dashboard-ingest/src/main/java/com/ingestpipeline/model/References.java @@ -0,0 +1,66 @@ +package com.ingestpipeline.model; + +import com.fasterxml.jackson.annotation.JsonProperty; + +public class References { + + + private String fieldName; + private String argument; + private String dataType; + private String value; + private String seperator; + private String expression; + + @JsonProperty(value="fieldName") + public String getFieldName() { + return fieldName; + } + + public void setFieldName(String fieldName) { + this.fieldName = fieldName; + } + + @JsonProperty(value="argument") + public String getArgument() { + return argument; + } + + public void setArgument(String argument) { + this.argument = argument; + } + + @JsonProperty(value="dataType") + public String getDataType() { + return dataType; + } + + public void setDataType(String dataType) { + this.dataType = dataType; + } + + @JsonProperty(value="value") + public String getValue() { + return value; + } + + public void setValue(String value) { + this.value = value; + } + @JsonProperty(value="seperator") + public String getSeperator() { + return seperator; + } + + public void setSeperator(String seperator) { + this.seperator = seperator; + } + @JsonProperty(value="expression") + public String getExpression() { + return expression; + } + + public void setExpression(String expression) { + this.expression = expression; + } +} diff --git a/dashboard-ingest/src/main/java/com/ingestpipeline/model/Response.java b/dashboard-ingest/src/main/java/com/ingestpipeline/model/Response.java new file mode 100644 index 000000000..84f69a280 --- /dev/null +++ b/dashboard-ingest/src/main/java/com/ingestpipeline/model/Response.java @@ -0,0 +1,29 @@ +package com.ingestpipeline.model; + +public class Response { + + private String message; + private String status; + + public Response(String status, String message){ + this.message = message; + this.status = status; + } + + + public String getMessage() { + return message; + } + + public void setMessage(String message) { + this.message = message; + } + + public String getStatus() { + return status; + } + + public void setStatus(String status) { + this.status = status; + } +} diff --git a/dashboard-ingest/src/main/java/com/ingestpipeline/model/Snippet.java b/dashboard-ingest/src/main/java/com/ingestpipeline/model/Snippet.java new file mode 100644 index 000000000..69d1e509c --- /dev/null +++ b/dashboard-ingest/src/main/java/com/ingestpipeline/model/Snippet.java @@ -0,0 +1,8 @@ +package com.ingestpipeline.model; + +public class Snippet { + public static void main(String[] args) { + + } +} + diff --git a/dashboard-ingest/src/main/java/com/ingestpipeline/model/SourceReferences.java b/dashboard-ingest/src/main/java/com/ingestpipeline/model/SourceReferences.java new file mode 100644 index 000000000..7e7fd4ae0 --- /dev/null +++ b/dashboard-ingest/src/main/java/com/ingestpipeline/model/SourceReferences.java @@ -0,0 +1,5 @@ +package com.ingestpipeline.model; + +public class SourceReferences extends References{ + +} diff --git a/dashboard-ingest/src/main/java/com/ingestpipeline/model/TargetData.java b/dashboard-ingest/src/main/java/com/ingestpipeline/model/TargetData.java new file mode 100644 index 000000000..168bc94cf --- /dev/null +++ b/dashboard-ingest/src/main/java/com/ingestpipeline/model/TargetData.java @@ -0,0 +1,148 @@ +package com.ingestpipeline.model; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.GeneratedValue; +import javax.persistence.GenerationType; +import javax.persistence.Id; +import javax.persistence.Table; +import java.io.Serializable; +import java.math.BigDecimal; + +@Entity +@Table(name = "TargetData") +@JsonIgnoreProperties +public class TargetData implements Serializable { + + @Id /*@GeneratedValue(strategy=GenerationType.AUTO) */ + @Column(name = "id", nullable = false) + private int id; + + @Column(name = "snoForMunicipalCorporation", nullable = false) + private String snoForMunicipalCorporation; + + @Column(name = "tenantIdForMunicipalCorporation", nullable = false) + private String tenantIdForMunicipalCorporation; + + @Column(name = "ulbName", nullable = false) + private String ulbName; + + @Column(name = "actualCollectionForMunicipalCorporation", nullable = false) + private BigDecimal actualCollectionForMunicipalCorporation; + + @Column(name = "budgetProposedForMunicipalCorporation", nullable = false) + private BigDecimal budgetProposedForMunicipalCorporation; + + @Column(name = "actualCollectionBudgetedForMunicipalCorporation", nullable = false) + private BigDecimal actualCollectionBudgetedForMunicipalCorporation; + + @Column(name = "financialYear", nullable = false) + private String financialYear; + + @Column(name = "Timestamp", nullable = false) + private String Timestamp; + + @Column(name = "businessService", nullable = false) + private String businessService; + + public TargetData() { + super(); + // TODO Auto-generated constructor stub + } + + @JsonProperty("id") + public int getId() { + return id; + } + @JsonProperty("id") + public void setId(int id) { + this.id = id; + } + @JsonProperty("snoForMunicipalCorporation") + public String getSnoForMunicipalCorporation() { + return snoForMunicipalCorporation; + } + + public void setSnoForMunicipalCorporation(String snoForMunicipalCorporation) { + this.snoForMunicipalCorporation = snoForMunicipalCorporation; + } + @JsonProperty("tenantIdForMunicipalCorporation") + public String getTenantIdForMunicipalCorporation() { + return tenantIdForMunicipalCorporation; + } + + public void setTenantIdForMunicipalCorporation(String tenantIdForMunicipalCorporation) { + this.tenantIdForMunicipalCorporation = tenantIdForMunicipalCorporation; + } + @JsonProperty("ulbName") + public String getUlbName() { + return ulbName; + } + @JsonProperty("ulbName") + public void setUlbName(String ulbName) { + this.ulbName = ulbName; + } + @JsonProperty("actualCollectionForMunicipalCorporation") + public BigDecimal getActualCollectionForMunicipalCorporation() { + return actualCollectionForMunicipalCorporation; + } + @JsonProperty("actualCollectionForMunicipalCorporation") + public void setActualCollectionForMunicipalCorporation(BigDecimal actualCollectionForMunicipalCorporation) { + this.actualCollectionForMunicipalCorporation = actualCollectionForMunicipalCorporation; + } + @JsonProperty("budgetProposedForMunicipalCorporation") + public BigDecimal getBudgetProposedForMunicipalCorporation() { + return budgetProposedForMunicipalCorporation; + } + @JsonProperty("budgetProposedForMunicipalCorporation") + public void setBudgetProposedForMunicipalCorporation(BigDecimal budgetProposedForMunicipalCorporation) { + this.budgetProposedForMunicipalCorporation = budgetProposedForMunicipalCorporation; + } + @JsonProperty("actualCollectionBudgetedForMunicipalCorporation") + public BigDecimal getActualCollectionBudgetedForMunicipalCorporation() { + return actualCollectionBudgetedForMunicipalCorporation; + } + @JsonProperty("actualCollectionBudgetedForMunicipalCorporation") + public void setActualCollectionBudgetedForMunicipalCorporation(BigDecimal actualCollectionBudgetedForMunicipalCorporation) { + this.actualCollectionBudgetedForMunicipalCorporation = actualCollectionBudgetedForMunicipalCorporation; + } + @JsonProperty("financialYear") + public String getFinancialYear() { + return financialYear; + } + @JsonProperty("financialYear") + public void setFinancialYear(String financialYear) { + this.financialYear = financialYear; + } + @JsonProperty("Timestamp") + public String getTimestamp() { + return Timestamp; + } + @JsonProperty("Timestamp") + public void setTimestamp(String timestamp) { + Timestamp = timestamp; + } + @JsonProperty("businessService") + public String getBusinessService() { + return businessService; + } + @JsonProperty("businessService") + public void setBusinessService(String businessService) { + this.businessService = businessService; + } + + @Override + public String toString() { + return "TargetData [id=" + id + ", snoForMunicipalCorporation=" + snoForMunicipalCorporation + + ", tenantIdForMunicipalCorporation=" + tenantIdForMunicipalCorporation + ", ulbName=" + ulbName + + ", actualCollectionForMunicipalCorporation=" + actualCollectionForMunicipalCorporation + + ", budgetProposedForMunicipalCorporation=" + budgetProposedForMunicipalCorporation + + ", actualCollectionBudgetedForMunicipalCorporation=" + actualCollectionBudgetedForMunicipalCorporation + + ", financialYear=" + financialYear + ", Timestamp=" + Timestamp + ", businessService=" + + businessService + "]"; + } + +} diff --git a/dashboard-ingest/src/main/java/com/ingestpipeline/model/TargetReferences.java b/dashboard-ingest/src/main/java/com/ingestpipeline/model/TargetReferences.java new file mode 100644 index 000000000..8e4984146 --- /dev/null +++ b/dashboard-ingest/src/main/java/com/ingestpipeline/model/TargetReferences.java @@ -0,0 +1,5 @@ +package com.ingestpipeline.model; + +public class TargetReferences extends References{ + +} diff --git a/dashboard-ingest/src/main/java/com/ingestpipeline/model/TopicContext.java b/dashboard-ingest/src/main/java/com/ingestpipeline/model/TopicContext.java new file mode 100644 index 000000000..f079c5753 --- /dev/null +++ b/dashboard-ingest/src/main/java/com/ingestpipeline/model/TopicContext.java @@ -0,0 +1,26 @@ +package com.ingestpipeline.model; + +public class TopicContext { + + private String topic; + private String dataContext; + private String dataContextVersion; + public String getTopic() { + return topic; + } + public void setTopic(String topic) { + this.topic = topic; + } + public String getDataContext() { + return dataContext; + } + public void setDataContext(String dataContext) { + this.dataContext = dataContext; + } + public String getDataContextVersion() { + return dataContextVersion; + } + public void setDataContextVersion(String dataContextVersion) { + this.dataContextVersion = dataContextVersion; + } +} diff --git a/dashboard-ingest/src/main/java/com/ingestpipeline/model/TopicContextConfig.java b/dashboard-ingest/src/main/java/com/ingestpipeline/model/TopicContextConfig.java new file mode 100644 index 000000000..5662f017a --- /dev/null +++ b/dashboard-ingest/src/main/java/com/ingestpipeline/model/TopicContextConfig.java @@ -0,0 +1,19 @@ +package com.ingestpipeline.model; + +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.util.ArrayList; +import java.util.List; + +public class TopicContextConfig { + + private List topicContexts = new ArrayList<>(); + + @JsonProperty(value = "topicContextConfigurations") + public List getTopicContexts() { + return topicContexts; + } + public void setTopicContexts(List topicContexts) { + this.topicContexts = topicContexts; + } +} diff --git a/dashboard-ingest/src/main/java/com/ingestpipeline/producer/HashMapSerializer.java b/dashboard-ingest/src/main/java/com/ingestpipeline/producer/HashMapSerializer.java new file mode 100644 index 000000000..b2840c3c3 --- /dev/null +++ b/dashboard-ingest/src/main/java/com/ingestpipeline/producer/HashMapSerializer.java @@ -0,0 +1,33 @@ +package com.ingestpipeline.producer; + +import java.util.Map; + +import org.apache.kafka.common.serialization.Serializer; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; + +public class HashMapSerializer implements Serializer { + @Override + public void close() { + // TODO Auto-generated method stub + } + + @Override + public void configure(Map arg0, boolean arg1) { + // TODO Auto-generated method stub + } + + @Override + public byte[] serialize(String topic, Map data) { + byte[] value = null; + ObjectMapper objectMapper = new ObjectMapper(); + try { + value = objectMapper.writeValueAsString(data).getBytes(); + } catch (JsonProcessingException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + return value; + } +} \ No newline at end of file diff --git a/dashboard-ingest/src/main/java/com/ingestpipeline/producer/IngestProducer.java b/dashboard-ingest/src/main/java/com/ingestpipeline/producer/IngestProducer.java new file mode 100644 index 000000000..46038684f --- /dev/null +++ b/dashboard-ingest/src/main/java/com/ingestpipeline/producer/IngestProducer.java @@ -0,0 +1,23 @@ +package com.ingestpipeline.producer; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.kafka.core.KafkaTemplate; +import org.springframework.stereotype.Service; + +import com.google.gson.Gson; + +@Service +public class IngestProducer { + + public static final Logger LOGGER = LoggerFactory.getLogger(IngestProducer.class); + + @Autowired + private KafkaTemplate kafkaTemplate; + + public void pushToPipeline(Object object, String topic, String key) { + LOGGER.info("Kafka Topic : " + topic + " Kafka Key : " + key); + kafkaTemplate.send(topic, key, object); + } +} diff --git a/dashboard-ingest/src/main/java/com/ingestpipeline/producer/IngestProducerConfig.java b/dashboard-ingest/src/main/java/com/ingestpipeline/producer/IngestProducerConfig.java new file mode 100644 index 000000000..a65fbbb33 --- /dev/null +++ b/dashboard-ingest/src/main/java/com/ingestpipeline/producer/IngestProducerConfig.java @@ -0,0 +1,49 @@ +package com.ingestpipeline.producer; + +import java.util.HashMap; +import java.util.Map; + +import org.apache.kafka.clients.producer.ProducerConfig; +import org.apache.kafka.common.serialization.StringSerializer; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.kafka.annotation.EnableKafka; +import org.springframework.kafka.core.DefaultKafkaProducerFactory; +import org.springframework.kafka.core.KafkaTemplate; +import org.springframework.kafka.core.ProducerFactory; +import org.springframework.kafka.support.serializer.JsonSerializer; + +@Configuration +@EnableKafka +public class IngestProducerConfig { + + @Value("${spring.kafka.bootstrap.servers}") + private String serverConfig; + + @Value("${kafka.producer.config.retries_config}") + private Integer retriesConfig; + + @Value("${kafka.producer.config.batch_size_config}") + private Integer batchSizeConfig; + + @Value("${kafka.producer.config.linger_ms_config}") + private Integer lingerMsConfig; + + @Value("${kafka.producer.config.buffer_memory_config}") + private Integer bufferMemoryConfig; + + @Bean + public ProducerFactory producerFactory() { + Map configProps = new HashMap<>(); + configProps.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, serverConfig); + configProps.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class); + configProps.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, JsonSerializer.class); + return new DefaultKafkaProducerFactory<>(configProps); + } + + @Bean + public KafkaTemplate kafkaTemplate() { + return new KafkaTemplate<>(producerFactory()); + } +} diff --git a/dashboard-ingest/src/main/java/com/ingestpipeline/producer/JavaSerializer.java b/dashboard-ingest/src/main/java/com/ingestpipeline/producer/JavaSerializer.java new file mode 100644 index 000000000..b99d21395 --- /dev/null +++ b/dashboard-ingest/src/main/java/com/ingestpipeline/producer/JavaSerializer.java @@ -0,0 +1,37 @@ +package com.ingestpipeline.producer; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.ObjectOutputStream; +import java.util.Map; + +import org.apache.kafka.common.serialization.Serializer; + +public class JavaSerializer implements Serializer { + + @Override + public byte[] serialize(String topic, Object data) { + try { + ByteArrayOutputStream byteStream = new ByteArrayOutputStream(); + ObjectOutputStream objectStream = new ObjectOutputStream(byteStream); + objectStream.writeObject(data); + objectStream.flush(); + objectStream.close(); + return byteStream.toByteArray(); + } + catch (IOException e) { + throw new IllegalStateException("Can't serialize object: " + data, e); + } + } + + @Override + public void configure(Map configs, boolean isKey) { + + } + + @Override + public void close() { + + } + +} \ No newline at end of file diff --git a/dashboard-ingest/src/main/java/com/ingestpipeline/repository/ElasticSearchRepository.java b/dashboard-ingest/src/main/java/com/ingestpipeline/repository/ElasticSearchRepository.java new file mode 100644 index 000000000..27a9d4a28 --- /dev/null +++ b/dashboard-ingest/src/main/java/com/ingestpipeline/repository/ElasticSearchRepository.java @@ -0,0 +1,71 @@ +package com.ingestpipeline.repository; + +import java.util.Map; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.http.HttpEntity; +import org.springframework.http.HttpHeaders; +import org.springframework.http.HttpMethod; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.stereotype.Service; +import org.springframework.web.client.HttpClientErrorException; +import org.springframework.web.client.HttpServerErrorException; +import org.springframework.web.client.RestTemplate; + +import com.ingestpipeline.util.Constants; + +/** + * This Repository Class is used to perform the transactions of storing the data into the Elastic Search Repository + * @author Darshan Nagesh + * + */ +@Service +public class ElasticSearchRepository { + + public static final Logger LOGGER = LoggerFactory.getLogger(ElasticSearchRepository.class); + + private final RestTemplate restTemplate; + + public ElasticSearchRepository(RestTemplate restTemplate) { + this.restTemplate = restTemplate; + } + + /** + * Based on the Transaction Index Data Obtained and the URL with Headers, this method will put the Data obtained on the + * Elastic Search Database and returns the response in the form of Positive or Negative outcome (True Or False) + * @param transactionIndex + * @param url + * @param headers + * @return + */ + public Boolean saveMyDataObject(Object object, String url, HttpHeaders headers) { + ResponseEntity map = null; + try { + map = restTemplate.exchange(url, HttpMethod.PUT, + new HttpEntity<>(object, headers), Map.class); + } catch (final HttpClientErrorException httpClientErrorException) { + LOGGER.error("Error : " + httpClientErrorException); + } catch (HttpServerErrorException httpServerErrorException) { + LOGGER.error("Error : " + httpServerErrorException); + } catch (Exception e) { + LOGGER.error("Error : " + e); + } + if (map != null && map.getStatusCode() != null && (map.getStatusCode() == HttpStatus.OK) || (map.getStatusCode() == HttpStatus.CREATED)) { + return true; + } + return false; + } + + public ResponseEntity fetchMDMSResponse(Object mdmsRequestObject) { + + HttpHeaders headers = new HttpHeaders(); + headers.set(Constants.CONTENT_TYPE, Constants.JSON); + + HttpEntity httpEntity = new HttpEntity<>(mdmsRequestObject, headers); + ResponseEntity result = restTemplate.exchange(Constants.MDMS_URL, HttpMethod.POST, httpEntity, Map.class); + + return result; + } + } diff --git a/dashboard-ingest/src/main/java/com/ingestpipeline/repository/TargetDataDao.java b/dashboard-ingest/src/main/java/com/ingestpipeline/repository/TargetDataDao.java new file mode 100644 index 000000000..50fea5867 --- /dev/null +++ b/dashboard-ingest/src/main/java/com/ingestpipeline/repository/TargetDataDao.java @@ -0,0 +1,18 @@ +package com.ingestpipeline.repository; + +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonSubTypes.Type; +import com.fasterxml.jackson.annotation.JsonTypeInfo; +import com.fasterxml.jackson.annotation.JsonTypeInfo.As; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.ingestpipeline.model.TargetData; +import org.springframework.data.repository.CrudRepository; +import org.springframework.stereotype.Repository; + +@Repository +public interface TargetDataDao extends CrudRepository { + +/* public List findAll(); + + public void save(TargetData targetData);*/ +} diff --git a/dashboard-ingest/src/main/java/com/ingestpipeline/service/CollectionTransformationService.java b/dashboard-ingest/src/main/java/com/ingestpipeline/service/CollectionTransformationService.java new file mode 100644 index 000000000..03d87f4b4 --- /dev/null +++ b/dashboard-ingest/src/main/java/com/ingestpipeline/service/CollectionTransformationService.java @@ -0,0 +1,233 @@ +package com.ingestpipeline.service; + +import com.bazaarvoice.jolt.Chainr; +import com.bazaarvoice.jolt.JsonUtils; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.ingestpipeline.util.ConfigLoader; +import com.ingestpipeline.util.Constants; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import java.io.ByteArrayInputStream; +import java.io.InputStream; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; + +@Service(Constants.Qualifiers.TRANSFORM_COLLECTION_SERVICE) +public class CollectionTransformationService implements TransformService { + + public static final Logger LOGGER = LoggerFactory.getLogger(CollectionTransformationService.class); + + + private static final String SEPARATOR = "_"; + private static final String JSON_EXTENSION = ".json"; + private static final String OBJECTIVE = "transform"; + private static final String CONFIGROOT = "config/"; + private static final String JOLT_SPEC = "spec"; + + private static final String TRANSACTION_ID = "transactionId"; + private static final String ID = "id"; + + + + @Autowired + private ConfigLoader configLoader; + + private String previousKey= ""; + + @Override + public Boolean transformData(Map incomingData) { + + LOGGER.info("incomingData at transformData :## "+incomingData.toString()); + Map incomingDataCopy = new HashMap<>(); + incomingDataCopy.putAll(incomingData); + incomingData.clear(); + + String dataContext = incomingDataCopy.get(Constants.DATA_CONTEXT).toString(); + String dataContextVersion = incomingDataCopy.get(Constants.DATA_CONTEXT_VERSION).toString(); + ObjectMapper mapper = new ObjectMapper(); + List chainrSpecJSON = null ; + + try { + + JsonNode incomingNode = mapper.convertValue(incomingDataCopy, JsonNode.class); + LOGGER.info("incoming data: "+incomingNode); + //JsonNode identifier = incomingNode.get(Constants.DATA_OBJECT).get(TRANSACTION_ID); + + + //To change: for loading the file from config root + String trsFile = OBJECTIVE.concat(SEPARATOR).concat(dataContext).concat(SEPARATOR).concat(dataContextVersion).concat(JSON_EXTENSION); + String strFile = configLoader.get(trsFile); + LOGGER.info("strFile:## "+strFile); + JsonNode specNode = mapper.readTree(strFile); + LOGGER.info("specNode:## "+specNode); + + + /*LOGGER.info("sourceUrl## "+strFile); + + String sourceUrl = CONFIGROOT.concat(OBJECTIVE.concat(SEPARATOR).concat(dataContext).concat(SEPARATOR).concat(dataContextVersion).concat(JSON_EXTENSION)); + + LOGGER.info("sourceUrl## "+sourceUrl); + + JsonNode specNode = mapper.readTree(this.getClass().getClassLoader().getResourceAsStream(sourceUrl)); + + LOGGER.info("specNode:## "+specNode);*/ + + //String sourceUrl = (OBJECTIVE.concat(SEPARATOR).concat(dataContext).concat(SEPARATOR).concat(dataContextVersion).concat(JSON_EXTENSION)); + //JsonNode specNode = mapper.readTree(configLoader.get(sourceUrl)); + JsonNode specData = specNode.findPath(JOLT_SPEC); + LOGGER.info("specData## " + specData); + + String previousField = findParentKey(specNode.findPath(JOLT_SPEC), "$i", ""); + LOGGER.info("previousField## " + previousField); + int parentNodeSize = incomingNode.findValues(previousField).get(0).size(); + LOGGER.info("parentNodeSize## " + parentNodeSize); + + for(int i=0; (i> fields = node.fields(); + while (fields.hasNext()) { + Map.Entry entry = fields.next(); + + if (entry.getKey().equalsIgnoreCase(value)) { + previousKey = key; + break; + } else if (entry.getValue().isObject()) { + key = entry.getKey(); + findParentKey(entry.getValue(), value, key); + } + } + return previousKey; + + } + + + /*public void transformData() { + + Map incomingData = new HashMap(); +*//* Map incomingDataCopy = new HashMap(); + incomingDataCopy.putAll(incomingData);*//* + incomingData.clear(); + //String dataContext = incomingData.get(Constants.DATA_CONTEXT).toString(); + //String dataContextVersion = incomingData.get(Constants.DATA_CONTEXT_VERSION).toString(); + ObjectMapper mapper = new ObjectMapper(); + List chainrSpecJSON = null ; + List incomingDataList = new ArrayList<>(); + + try { + + JsonNode specNode = mapper.readTree(configLoader.get("anytransform_collection_v2.json")); + + JsonNode incoming = mapper.readTree(configLoader.get("anyinput.json"));//TODO: use incomming node object + Map incommingMap = new HashMap(); + incommingMap = mapper.convertValue(incoming, new TypeReference>(){}); + + String previousField = findKey(specNode.findPath(JOLT_SPEC), "$i", ""); + int parentNodeSize = incoming.findValues(previousField).get(0).size(); + + for(int i=0; (i productList = null; // getBiProducts(incomingData, digressionPoint); + /*if(productList != null && productList.size() > 0) { + for(JsonNode product : productList) { + Map subData = new HashMap<>(); + subData.put(Constants.DATA_CONTEXT, incomingData.getDataContext()); + subData.put(Constants.DATA_CONTEXT_VERSION, incomingData.getDataContextVersion()); + subData.put(Constants.DATA_OBJECT, product); + } + }*/ + return null; + } + + private Object ingestServiceReplace(Map incomingData) { + + return null; + } + + /*private List getBiProducts(IncomingData incomingData, DigressionPoint point) { + Object dataObject = incomingData.getDataObject(); + List dataObjectList = new ArrayList<>(); + List paths = point.getPaths(); + ObjectMapper mapper = new ObjectMapper(); + Gson gson = new Gson(); + JsonNode mainNode = null; + try { + mainNode = mapper.readTree(gson.toJson(dataObject)); + } catch (Exception ex) { + LOGGER.error("Encountered an error while converting the data to JSON Node : " + ex.getMessage()); + } + List list = generateBiProducts(point.getPaths(), mainNode); + return list; + } + + private List generateBiProducts(List paths, JsonNode mainNode) { + List jsonNodeList = new ArrayList<>(); + for(Path path : paths) { + jsonNodeList = getSplitObjects(path.getRoute(), mainNode, path.getRemoveReplace()); + for(JsonNode subNode : jsonNodeList) { + if(path.getPaths() != null && path.getPaths().size() > 0) { + generateBiProducts(path.getPaths(), subNode); + } + } + } + if(jsonNodeList.size() == 0) { + jsonNodeList.add(mainNode); + } + return jsonNodeList; + } + + private List getSplitObjects(String route, JsonNode mainNode, String removeReplace) { + List nodeList = new ArrayList<>(); + String[] routes = route.split("\\."); + List routesList = Arrays.asList(routes); + JsonNode node = mainNode; + for(String r : routes) { + node = node.path(r); + } + if(node.isArray()) { + ArrayNode arrayNode = (ArrayNode) node; + if(arrayNode.size() > 1) { + ObjectNode finalSubNode = null; + for(int i=0 ; i < arrayNode.size() ; i++) { + ObjectNode subNode = mainNode.deepCopy(); + JsonNode subJsonNode = mainNode.deepCopy(); + for(String r : routes) { + if(subNode.get(r).deepCopy().isArray()) { + ArrayNode arraySubNode = subNode.get(r).deepCopy(); + subNode = arraySubNode.get(i).deepCopy(); + } else { + JsonNode jsonSubNode = subNode.get(r).deepCopy(); + subNode = jsonSubNode.deepCopy(); + } + } + ObjectNode mainObjectNode = mainNode.deepCopy(); + for(String r : routes) { + if(!r.equals(removeReplace)) { + mainObjectNode = mainObjectNode.get(r).deepCopy(); + } else { + mainObjectNode.remove(removeReplace); + routesList.remove(removeReplace); + } + } + for(int j =0 ; j caughtFailedRequests = new HashMap<>(); + + + @Value("${es.index.type}") + public String DOC_PATH; + + @Value("${services.esindexer.host}") + private String indexServiceHost; + @Value("${egov.services.esindexer.host.search}") + private String indexServiceHostSearch; + + @Value("${es.host.schema}") + private String schema; + + @Value("${services.esindexer.host}") + private String indexerServiceHost; + @Value("${es.target.index.name}") + private String targetIndexName; + + @Value("${es.index.name}") + private String collectionIndexName; + + @Value("${es.index.searchQuery.collection}") + private String searchQueryCollection; + + @Value("${es.index.searchQuery.billing}") + private String searchQueryBilling; + + @Value("${es.index.searchQuery.payment}") + private String searchQueryPayment; + + @Autowired + private RestTemplate restTemplate; + + @Autowired + private RetryTemplate retryTemplate; + + @Autowired + private IngestService ingestService; + + private static final String SLASH_SEPERATOR = "/"; + + public static final Logger LOGGER = LoggerFactory.getLogger(ElasticService.class); + + public String getSearchQueryCollection() { + return searchQueryCollection; + } + + public void setSearchQueryCollection(String searchQueryCollection) { + this.searchQueryCollection = searchQueryCollection; + } + + public String getSearchQueryPayment() { + return searchQueryPayment; + } + + public void setSearchQueryPayment(String searchQueryPayment) { + this.searchQueryPayment = searchQueryPayment; + } + + public String getSearchQueryBilling() { + return searchQueryBilling; + } + + public void setSearchQueryBilling(String searchQueryBilling) { + this.searchQueryBilling = searchQueryBilling; + } + + + public ResponseEntity post(String index, String type, String id, String authToken, String requestNode) { + + StringBuilder uriBuilder = new StringBuilder(indexerServiceHost.concat(index).concat(SLASH_SEPERATOR).concat(type).concat(SLASH_SEPERATOR).concat(id)); + HttpHeaders headers = new HttpHeaders(); + if(authToken != null && !authToken.isEmpty()) + headers.add("Authorization", "Bearer "+ authToken ); + headers.setContentType(MediaType.APPLICATION_JSON); + + HttpEntity requestEntity = null; + if(requestNode != null ) requestEntity = new HttpEntity<>(requestNode, headers); + else requestEntity = new HttpEntity<>("{}", headers); + + ResponseEntity response = new ResponseEntity(HttpStatus.OK); + + try { + response = retryTemplate.postForEntity(uriBuilder.toString(), requestEntity); + //restTemplate.postForEntity(uri,requestEntity); + LOGGER.info("RestTemplate response status :: {}", response.getStatusCode()); + + } catch (HttpClientErrorException e) { + response = new ResponseEntity(e.getStatusCode()); + LOGGER.error("post client exception: " + e.getMessage()); + } + return response; + } + + @Override + public JsonNode search(String index, ObjectNode searchQuery) throws IOException { + SearchRequest searchRequest = buildSearchRequest(index, searchQuery); + SearchResponse searchResponse = getClient(index, indexServiceHost, 9200, schema).search(searchRequest, + RequestOptions.DEFAULT); + + ArrayNode resultArray = JsonNodeFactory.instance.arrayNode(); + for (SearchHit hit : searchResponse.getHits()) { + JsonNode node = new ObjectMapper().readValue(hit.getSourceAsString(), JsonNode.class); + resultArray.add(node); + } + return resultArray; + } + + @Override + public Map search(String index, String searchQuery) throws Exception { + + String url = indexServiceHost + index + indexServiceHostSearch; + HttpHeaders headers = new HttpHeaders(); + headers.setContentType(MediaType.APPLICATION_JSON); + + LOGGER.info("searching ES for query::" + searchQuery + "::on::" + index + "::ON URL::" + url); + + HttpEntity requestEntity = new HttpEntity<>(searchQuery, headers); + + try { + ResponseEntity response = retryTemplate.postForEntity(url,requestEntity); + + Map responseNode = new ObjectMapper().convertValue(response.getBody(), Map.class); + Map hits = (Map)responseNode.get("hits"); + if((Integer)hits.get("total") >=1) + return (Map)((ArrayList)hits.get("hits")).get(0); + + } catch (HttpClientErrorException e) { + e.printStackTrace(); + LOGGER.error("client error while searching ES : " + e.getMessage()); + + } + return null; + } + + @Override + public Boolean push(Map requestBody) throws Exception { + + Object id = requestBody.get(Constants.IDENTIFIER); + Object trxid = ((Map)requestBody.get(Constants.DATA_OBJECT)).get(Constants.ID); + LOGGER.info("request body on ### trxid " +trxid); + //Object dataObjectId = ((Map)requestBody.get(Constants.DATA_OBJECT)).get("id"); + + + + String docId = trxid!=null ? trxid.toString() : id.toString() ; + StringBuilder url = new StringBuilder().append(indexerServiceHost).append(collectionIndexName).append(SLASH_SEPERATOR).append(DOC_PATH).append(SLASH_SEPERATOR).append(docId); + LOGGER.info("url ## " +url); + + + HttpHeaders headers = new HttpHeaders(); + headers.setContentType(MediaType.APPLICATION_JSON); + + LOGGER.info("Posting request to ES on ::" + collectionIndexName + " with doc id:: "+docId); + + JsonNode request = new ObjectMapper().convertValue(requestBody, JsonNode.class); + //LOGGER.info(" new request body json ### " +request); + + HttpEntity requestEntity = new HttpEntity<>(request.toString(), headers); + ArrayNode hitNodes = null; + + try { + ResponseEntity response = retryTemplate.postForEntity(url.toString(), requestEntity); + LOGGER.info("Status code on pushing to collection index : {}", response.getStatusCode()); + if (response.getStatusCode().value() == HttpStatus.CREATED.value()) + return Boolean.TRUE; + + } catch (HttpClientErrorException e) { + //e.printStackTrace(); + + //caughtFailedRequests.put(dataObjectId.toString(), request); + LOGGER.error("client error while pushing ES collection index : {}, {}" ,e.getMessage(), requestEntity.getBody()); + + } + return Boolean.FALSE; + } + + @Override + public Boolean push(TargetData requestBody) throws Exception { + + Long currentDateTime = new Date().getTime(); + String url = indexerServiceHost + targetIndexName + DOC_TYPE + requestBody.getId(); + + HttpHeaders headers = new HttpHeaders(); + headers.setContentType(MediaType.APPLICATION_JSON); + + LOGGER.info("Posting request to ES on ## " + targetIndexName); + LOGGER.info("request body on ### " +requestBody); + + JsonNode request = new ObjectMapper().convertValue(requestBody, JsonNode.class); + + HttpEntity requestEntity = new HttpEntity<>(request.toString(), headers); + ArrayNode hitNodes = null; + + try { + ResponseEntity response = retryTemplate.postForEntity(url, requestEntity); + + //ResponseEntity response = restTemplate.exchange(url, HttpMethod.PUT, requestEntity, Object.class); + LOGGER.info("Status code on pushing to target index : " + response.getStatusCode()); + if (response.getStatusCode().value() == 201) + return Boolean.TRUE; + + } catch (HttpClientErrorException e) { + e.printStackTrace(); + LOGGER.error("client error while pushing ES target index : " + e.getMessage()); + + } + return Boolean.FALSE; + } + + @Override + public Boolean searchIndex(String index, String searchQuery, String dataContextVersion) throws Exception { + LOGGER.info("searching ES for query: " + searchQuery + " on " + index); + + Map scrollSearchParams = getScrollIdForScrollSearch(index, dataContextVersion); + LOGGER.info("scrollSearchParams.get(Constants.DataContexts.CONTEXT):: "+scrollSearchParams.get(Constants.DataContexts.CONTEXT)); + + if(scrollSearchParams == null) { + return Boolean.FALSE; + } + new Thread(new Runnable() { + public void run(){ + Map> documentMap = new HashMap<>(); + int docFetchSize = 1; + int totalDocFetchedSize = 0; + int totalDocInIndex = 2500000; + + while(docFetchSize > 0 && totalDocFetchedSize < totalDocInIndex) { + documentMap = performScrollSearch(scrollSearchParams); + List listOfDocs = documentMap.get("hits"); + docFetchSize = listOfDocs.size(); + totalDocFetchedSize = docFetchSize + totalDocFetchedSize; + for (Map.Entry> entry : documentMap.entrySet()) { + + for (int i = 0; i < entry.getValue().size(); i++) { + Map innerMap = (Map) entry.getValue().get(i); + Gson gson = new Gson(); + String json = gson.toJson(innerMap.get("_source")); + ObjectMapper mapper = new ObjectMapper(); + JsonNode dataNode = null; + try { + dataNode = mapper.readTree(json); + } catch (IOException e) { + LOGGER.error("Encountered an exception while reading the JSON Node on Thread : " + e.getMessage()); + } + JsonNode dataObjectNode = null; + if(dataNode != null && dataNode.get("Data") == null) { + dataObjectNode = dataNode; + } else { + dataObjectNode= dataNode.get("Data"); + } + Map dataMap = new Gson().fromJson(dataObjectNode.toString(), new TypeToken>() {}.getType() + ); + ingestService.ingestToPipeline( + setIncomingData(scrollSearchParams.get(Constants.DataContexts.CONTEXT), dataContextVersion, dataObjectNode)); + } + } + } + } + }).start(); + + + return Boolean.TRUE; + } + + @Override + public List searchMultiple(String index, String searchQuery) throws Exception { + + String url = indexServiceHost + index + indexServiceHostSearch; + HttpHeaders headers = new HttpHeaders(); + headers.setContentType(MediaType.APPLICATION_JSON); + + LOGGER.info("searching searchMultiple ES for query::" + searchQuery + "::on::" + index + "::ON URL::" + url); + + HttpEntity requestEntity = new HttpEntity<>(searchQuery, headers); + + try { + ResponseEntity response = retryTemplate.postForEntity(url,requestEntity); + + Map responseNode = new ObjectMapper().convertValue(response.getBody(), Map.class); + Map hits = (Map)responseNode.get("hits"); + if((Integer)hits.get("total") >=1) + return (List) ((ArrayList)hits.get("hits")); + + } catch (HttpClientErrorException e) { + e.printStackTrace(); + LOGGER.error("client error while searchMultiple searching ES : " + e.getMessage()); + + } + return null; + } + + private IncomingData setIncomingData(String index, String version, Object documentValue) { + IncomingData incomingData = new IncomingData(); + incomingData.setDataContext(index); + incomingData.setDataContextVersion(version); + incomingData.setDataObject(documentValue); + return incomingData; + } + + private Map performScrollSearch(Map scrollSearchParams) { + Map> hitsToMap = new LinkedHashMap(); + try { + Map hits = new LinkedHashMap(); + HttpEntity requestEntity = new HttpEntity<>(scrollSearchParams.get(Constants.ScrollSearch.QUERY), getHttpHeaders()); + ResponseEntity response = restTemplate.exchange(scrollSearchParams.get(Constants.ScrollSearch.SEARCH_PATH), HttpMethod.POST, requestEntity, + Object.class); + Map responseNode = new ObjectMapper().convertValue(response.getBody(), Map.class); + hits = (Map) responseNode.get("hits"); + if ((Integer) hits.get("total") >= 1) { + hitsToMap.put("hits", ((ArrayList) hits.get("hits"))); + return hitsToMap; + } + } catch (HttpClientErrorException e) { + LOGGER.error("client error while searching ES : " + e.getMessage()); + } + return hitsToMap; + } + + private Map getScrollIdForScrollSearch(String index, String dataContextVersion) { + Map scrollSearchParams = new HashMap<>(); + String queryString = null; + if (index.equals(Constants.ES_INDEX_COLLECTION)) { + queryString = getSearchQueryCollection(); + scrollSearchParams.put(Constants.DataContexts.CONTEXT, Constants.DataContexts.COLLECTION); + } else if (index.equals(Constants.ES_INDEX_BILLING)) { + queryString = getSearchQueryBilling(); + scrollSearchParams.put(Constants.DataContexts.CONTEXT, Constants.DataContexts.BILLING); + } else if (index.equals(Constants.ES_INDEX_PAYMENT)) { + queryString = getSearchQueryPayment(); + scrollSearchParams.put(Constants.DataContexts.CONTEXT, Constants.DataContexts.COLLECTION); + } else { + return null; + } + String scrollUrl = indexServiceHost + index + indexServiceHostSearch + "?scroll=1m"; + HttpEntity requestEntity = new HttpEntity<>(queryString, getHttpHeaders()); + String str = null; + try { + LOGGER.info("Request Body for Scroll Search : "+requestEntity); + ResponseEntity response = restTemplate.exchange(scrollUrl, HttpMethod.POST, requestEntity, + Object.class); + Map responseNode = new ObjectMapper().convertValue(response.getBody(), Map.class); + + Map> hitsToMap = new LinkedHashMap(); + Map hits = new LinkedHashMap(); + hits = (Map) responseNode.get("hits"); + if ((Integer) hits.get("total") >= 1) { + hitsToMap.put("hits", ((ArrayList) hits.get("hits"))); + } + + for (Entry> entry : hitsToMap.entrySet()) { + + for (int i = 0; i < entry.getValue().size(); i++) { + Map innerMap = (Map) entry.getValue().get(i); + Gson gson = new Gson(); + String json = gson.toJson(innerMap.get("_source")); + ObjectMapper mapper = new ObjectMapper(); + JsonNode dataNode = null; + try { + dataNode = mapper.readTree(json); + } catch (IOException e) { + LOGGER.error("Encountered an exception while reading the JSON Node on Thread : " + e.getMessage()); + } + JsonNode dataObjectNode = null; + if(dataNode != null && dataNode.get("Data") == null) { + dataObjectNode = dataNode; + } else { + dataObjectNode= dataNode.get("Data"); + } + Map dataMap = new Gson().fromJson(dataObjectNode.toString(), new TypeToken>() {}.getType() + ); + ingestService.ingestToPipeline( + setIncomingData(scrollSearchParams.get(Constants.DataContexts.CONTEXT), dataContextVersion, dataObjectNode)); + } + } + str = indexServiceHostSearch.replaceAll("[/]", ""); + scrollSearchParams.put(Constants.ScrollSearch.SCROLL_ID, (String) responseNode.get(Constants.ScrollSearch.SCROLL_ID_PARAMS)); + scrollSearchParams.put(Constants.ScrollSearch.SEARCH_PATH, indexServiceHost + str + "/" + "scroll"); + String queryForScrollId = Constants.ScrollSearch.SCROLL_SEARCH_DEFAULT_QUERY + "\"" + scrollSearchParams.get(Constants.ScrollSearch.SCROLL_ID) + "\"" + "}"; + scrollSearchParams.put(Constants.ScrollSearch.QUERY, queryForScrollId); + } catch (HttpClientErrorException e) { + LOGGER.error("client error while searching ES : " + e.getMessage()); + } + return scrollSearchParams; + } + + + + private HttpHeaders getHttpHeaders() { + HttpHeaders headers = new HttpHeaders(); + headers.setContentType(MediaType.APPLICATION_JSON); + return headers; + } + + +} diff --git a/dashboard-ingest/src/main/java/com/ingestpipeline/service/EnrichTransform.java b/dashboard-ingest/src/main/java/com/ingestpipeline/service/EnrichTransform.java new file mode 100644 index 000000000..cbcc4641c --- /dev/null +++ b/dashboard-ingest/src/main/java/com/ingestpipeline/service/EnrichTransform.java @@ -0,0 +1,106 @@ +package com.ingestpipeline.service; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import com.bazaarvoice.jolt.Chainr; +import com.bazaarvoice.jolt.JsonUtils; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.ingestpipeline.util.ConfigLoader; + +@Service +public class EnrichTransform { + + public static final Logger LOGGER = LoggerFactory.getLogger(EnrichTransform.class); + + @Autowired + private TransformService transformService; + + @Autowired + private ConfigLoader configLoader; + + private static final String SEPARATOR = "_"; + private static final String JSON_EXTENSION = ".json"; + private static final String OBJECTIVE = "transform"; + private static final String CONFIGROOT = "config/"; + private static final String VERSION = "v1"; + private static final String ENHANCE = "enhance"; + + + + /** + * Tranforms domain raw response from elastic search + * This transformation is specific to domain objects + * + * @param rawResponseNode + * @param businessService + * @return + */ + public Object transform (Map rawResponseNode, String businessService) throws IOException { + + ObjectMapper mapper = new ObjectMapper(); + List chainrSpecJSON = null ; + Object transNode = null; + try { + chainrSpecJSON = mapper.readValue(configLoader.get(OBJECTIVE.concat(SEPARATOR).concat(businessService.toLowerCase()).concat(SEPARATOR).concat(VERSION).concat(JSON_EXTENSION)), List.class); + LOGGER.info("ChainrSpecJSON::" + chainrSpecJSON); + Chainr chainr = Chainr.fromSpec( chainrSpecJSON ); + + Object indexData = rawResponseNode.keySet().contains("_source") ? ((Map)rawResponseNode.get("_source")).get("Data") : null; + transNode = indexData!= null ? chainr.transform(indexData) : null; + + } catch (Exception e) { + LOGGER.error("Encountered an error : businessService {} ", e.getMessage()); + } + + return transNode; + + } + + + /** + * Tranforms domain raw response from elastic search + * This transformation is specific to domain objects + * + * @param rawResponseNode + * @param businessService + * @return + */ + public Object transformEnhanceData (List rawResponseList, String businessService) throws IOException { + + ObjectMapper mapper = new ObjectMapper(); + List chainrSpecJSON = null ; + Object transNode = null; + List response = new ArrayList(); + + try { + chainrSpecJSON = mapper.readValue(configLoader.get(OBJECTIVE.concat(SEPARATOR).concat(businessService.toLowerCase()).concat(SEPARATOR).concat(ENHANCE).concat(SEPARATOR).concat(VERSION).concat(JSON_EXTENSION)), List.class); + LOGGER.info("ChainrSpecJSON::" + chainrSpecJSON); + Chainr chainr = Chainr.fromSpec( chainrSpecJSON ); + + for (Object object : rawResponseList) { + if(object instanceof Map) { + Map rawResponseNode = (Map) object; + Object indexData = rawResponseNode.keySet().contains("_source") ? ((Map)rawResponseNode.get("_source")).get("Data") : null; + transNode = indexData!= null ? chainr.transform(indexData) : null; + response.add(transNode); + } + + } + + } catch (Exception e) { + LOGGER.error("Encountered in transformEnhanceData with an error : businessService {} ", e.getMessage()); + } + + return response; + + } + +} diff --git a/dashboard-ingest/src/main/java/com/ingestpipeline/service/EnrichmentService.java b/dashboard-ingest/src/main/java/com/ingestpipeline/service/EnrichmentService.java new file mode 100644 index 000000000..6f14ff224 --- /dev/null +++ b/dashboard-ingest/src/main/java/com/ingestpipeline/service/EnrichmentService.java @@ -0,0 +1,14 @@ +package com.ingestpipeline.service; + +import org.springframework.stereotype.Service; + +import com.ingestpipeline.model.IncomingData; + +import java.util.Map; + +@Service +public interface EnrichmentService { + + Map enrichData(Map incomingData); + +} diff --git a/dashboard-ingest/src/main/java/com/ingestpipeline/service/EnrichmentServiceImpl.java b/dashboard-ingest/src/main/java/com/ingestpipeline/service/EnrichmentServiceImpl.java new file mode 100644 index 000000000..835eb7bdc --- /dev/null +++ b/dashboard-ingest/src/main/java/com/ingestpipeline/service/EnrichmentServiceImpl.java @@ -0,0 +1,324 @@ +package com.ingestpipeline.service; + +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import com.ingestpipeline.model.SourceReferences; +import com.ingestpipeline.model.TargetReferences; +import org.apache.tomcat.util.codec.binary.Base64; +import org.json.JSONObject; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.http.HttpHeaders; +import org.springframework.http.MediaType; +import org.springframework.stereotype.Service; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.google.common.hash.Hashing; +import com.ingestpipeline.config.DomainConfig; +import com.ingestpipeline.config.DomainConfigFactory; +import com.ingestpipeline.configfactory.CollectionDomainConfig; +import com.ingestpipeline.configfactory.EnhanceDomainConfig; +import com.ingestpipeline.model.DomainIndexConfig; +import com.ingestpipeline.model.TargetData; +import com.ingestpipeline.repository.ElasticSearchRepository; +import com.ingestpipeline.repository.TargetDataDao; +import com.ingestpipeline.util.Constants; +import com.ingestpipeline.util.JSONUtil; + +/** + * This is a Service Implementation for all the actions which are with respect + * to Elastic Search + * + * @author Darshan Nagesh + * + */ +@Service(Constants.Qualifiers.ENRICHMENT_SERVICE) +public class EnrichmentServiceImpl implements EnrichmentService { + + public static final Logger LOGGER = LoggerFactory.getLogger(EnrichmentServiceImpl.class); + private static final String SEPARATOR = "_"; + private static final String JSON_EXTENSION = ".json"; + private static final String OBJECTIVE = "enrichment"; + private final String indexServiceHost; + private final String userName; + private final String password; + private final String elasticSearchIndexName; + private final String elasticSearchDocumentType; + + private static final String AUTHORIZATION = "Authorization"; + private static final String US_ASCII = "US-ASCII"; + private static final String BASIC_AUTH = "Basic %s"; + + private static final String BUSINESS_SERVICE = "businessService"; + private static final String DATA_OBJECT = "dataObject"; + private static final String DATA_CONTEXT = "dataContext"; + private static final String DATA_ENHANCEMENT = "dataEnhancement"; + private static final String TENANTID = "tenantId"; + private static final String MCOLLECT = "MCOLLECT"; + + @Autowired + private ElasticSearchRepository elasticRepository; + + @Autowired + private CollectionDomainConfig collectionDomainConfig; + + @Autowired + private DomainConfigFactory domainConfigFactory; + + @Autowired + private IESService elasticService; + + @Autowired + private TargetDataDao targetDataDao; + + @Autowired + private EnrichTransform enrichTransform; + + @Autowired + private JSONUtil util; + + public EnrichmentServiceImpl(@Value("${services.esindexer.host}") String indexServiceHost, + @Value("${services.esindexer.username}") String userName, + @Value("${services.esindexer.password}") String password, + @Value("${es.index.name}") String elasticSearchIndexName, + @Value("${es.document.type}") String elasticSearchDocumentType, JSONUtil util) { + this.indexServiceHost = indexServiceHost; + this.userName = userName; + this.password = password; + this.elasticSearchIndexName = elasticSearchIndexName; + this.elasticSearchDocumentType = elasticSearchDocumentType; + this.util=util; + } + + @Override + public Map enrichData(Map incomingData) { + + Map dataObject = new ObjectMapper().convertValue(incomingData.get(DATA_OBJECT), Map.class); + LOGGER.info("data object at enrichData"+dataObject.toString()); + String tenantId = dataObject.get(TENANTID).toString(); + + List mCollectCategories = util.fetchMCollectCategories(tenantId); + + DomainConfig domainConfig = domainConfigFactory.getConfiguration(incomingData.get(DATA_CONTEXT).toString()); + LOGGER.info("domainConfig ## "+domainConfig); + + if(domainConfig instanceof CollectionDomainConfig) { + // prepare the query required based on incoming data businessType + ObjectNode incomingNode = new ObjectMapper().convertValue(incomingData.get(DATA_OBJECT), ObjectNode.class); + ObjectNode copyNode = incomingNode.deepCopy(); + String businessTypeVal = copyNode.findValue(BUSINESS_SERVICE).asText(); + + for(String category : mCollectCategories) { + if(category.equalsIgnoreCase(businessTypeVal)) { + businessTypeVal = MCOLLECT; + break; + } + } + + DomainIndexConfig indexConfig = domainConfig.getIndexConfig(businessTypeVal.toString()); + LOGGER.info("indexConfig ## "+indexConfig); + if(indexConfig != null){ + String indexName = indexConfig.getIndexName(); + + String query = indexConfig.getQuery(); + + try { + ObjectNode queryNode = new ObjectMapper().readValue(query, ObjectNode.class); + + Map expValMap = new HashMap<>(); + // Source references to be prepare a map of fieldName & value + for (SourceReferences ref : indexConfig.getSourceReferences()){ + String arg = ref.getFieldName(); + String argVal = copyNode.findValue(arg).asText(); + String[] values = argVal.split(ref.getSeperator()); + + String[] exps = ref.getExpression().split(ref.getSeperator()); + + for(int i=0; i { + TargetData targetData = new ObjectMapper().convertValue(val, TargetData.class); + String hashId = targetData.getFinancialYear() + "-" + targetData.getBusinessService()+"-"+targetData.getUlbName(); + hashId = hashId.replaceAll("(\\s)+", ""); + String sha256hex = Hashing.sha256().hashString(hashId, StandardCharsets.UTF_8).toString(); + targetData.setId(sha256hex.hashCode()); + //targetData.setId(hashId); + try{ + elasticService.push(targetData); + }catch (Exception e ){ + LOGGER.error("Exception occurred while pushing data to ES: " + e.getMessage()); + } + + }); + } + + return incomingData; + } + + private Boolean pushToElasticSearchIndex(Object object) { + Long currentDateTime = new Date().getTime(); + String url = String.format("%s%s/%s/%s", this.indexServiceHost, elasticSearchIndexName, + elasticSearchDocumentType, currentDateTime); + HttpHeaders headers = getHttpHeaders(); + LOGGER.info("Data Object to be added to ES : " + object); + LOGGER.info("URL to invoke : " + url); + elasticRepository.saveMyDataObject(object, url, headers); + return Boolean.TRUE; + } + + /** + * A helper method to create the headers for Rest Connection with UserName and + * Password + * + * @return HttpHeaders + */ + private HttpHeaders getHttpHeaders() { + HttpHeaders headers = new HttpHeaders(); + headers.add(AUTHORIZATION, getBase64Value(userName, password)); + headers.setContentType(MediaType.APPLICATION_FORM_URLENCODED); + headers.setContentType(MediaType.APPLICATION_JSON); + return headers; + } + + /** + * Helper Method to create the Base64Value for headers + * + * @param userName + * @param password + * @return + */ + private String getBase64Value(String userName, String password) { + String authString = String.format("%s:%s", userName, password); + byte[] encodedAuthString = Base64.encodeBase64(authString.getBytes(Charset.forName(US_ASCII))); + return String.format(BASIC_AUTH, new String(encodedAuthString)); + } + + /** + * This method used to add the external index data into the es index + * + * @param incomingData + * @return + */ + private Map enhanceData(Map incomingData) { + DomainConfig domainConfig = domainConfigFactory.getConfiguration(DATA_ENHANCEMENT); + LOGGER.info("enhanceDomainConfig ## "+domainConfig); + + if(domainConfig instanceof EnhanceDomainConfig ) { + // prepare the query required based on incoming data businessType + ObjectNode incomingNode = new ObjectMapper().convertValue(incomingData.get(DATA_OBJECT), ObjectNode.class); + ObjectNode copyNode = incomingNode.deepCopy(); + String businessTypeVal = copyNode.findValue(BUSINESS_SERVICE).asText(); + + DomainIndexConfig indexConfig = domainConfig.getIndexConfig(businessTypeVal.toString()); + LOGGER.info("indexConfig ## "+indexConfig); + if(indexConfig != null){ + String indexName = indexConfig.getIndexName(); + String query = indexConfig.getQuery(); + + + try { + ObjectNode queryNode = new ObjectMapper().readValue(query, ObjectNode.class); + LOGGER.info("queryNode 114 ## "+queryNode.toString()); + + Map expValMap = new HashMap<>(); + // Source references to be prepare a map of fieldName & value + for (SourceReferences ref : indexConfig.getSourceReferences()){ + String arg = ref.getFieldName(); + LOGGER.info("arg350## " + arg); + String argVal = copyNode.findValue(arg).asText(); + String[] values = argVal.split(ref.getSeperator()); + String[] exps = ref.getExpression().split(ref.getSeperator()); + + for(int i=0; i post(String index, String type, String id, String authToken, String requestNode); + + /** + * Holds client for each indices. + */ + static Map esClient = new HashMap(); + static final String INVALID_QUERY_ERROR ="invalid query: must be boolean, match_phrase "; + + /** searches documents from ES based on query using RestHighLevelClient + * @param index - ElasticSearch Index + * @param searchQuery - which contains details for search + * @return + */ + JsonNode search(String index, ObjectNode searchQuery) throws IOException; + + Boolean push(TargetData requestBody) throws Exception; + + /** + * searches documents from ES based on query using restTemplate + * @param index + * @param query + * @return + * @throws Exception + */ + Map search(String index, String query) throws Exception; + + List searchMultiple(String index, String query) throws Exception; + Boolean push(Map requestBody) throws Exception; + + + /** + * Translates a string query to SearchRequest + * valid query string has bool type query + * @param queryNode + * @return + */ + default SearchRequest buildSearchRequest(String index, ObjectNode queryNode) throws IOException{ + final BoolQueryBuilder query = QueryBuilders.boolQuery(); + + ArrayNode mustNodeArr = (ArrayNode) queryNode.get("query").get("bool").get("must"); + if(null == mustNodeArr) + throw new IllegalArgumentException(INVALID_QUERY_ERROR); + + mustNodeArr.elements().forEachRemaining(mustNode -> { + mustNode.fields().forEachRemaining(entry -> { + if(entry.getKey().equalsIgnoreCase("match_phrase") && entry.getValue().isObject()){ + JsonNode node = entry.getValue(); + Map.Entry dataNode = node.fields().next(); + query.must(QueryBuilders.matchPhraseQuery(dataNode.getKey(), dataNode.getValue().textValue())); + } + }); + + }); + return new SearchRequest(index).source(new SearchSourceBuilder().query(query)); + } + + /** + * + * @param indexName + * @param hostName + * @param port + * @param schema + * @return + */ + default RestHighLevelClient getClient(String indexName, String hostName, Integer port, String schema){ + RestHighLevelClient client = esClient.get(indexName); + if(null == client){ + client = new RestHighLevelClient(RestClient.builder(new HttpHost(hostName, port, schema))); + esClient.put(indexName, client); + } + return client; + } + + /*default boolean isIndexExists(String indexName) { + Response response; + try { + response = getClient(indexName).getLowLevelClient().performRequest(new Request("HEAD", "/" + indexName)); + return (200 == response.getStatusLine().getStatusCode()); + } catch (IOException e) { + return false; + } + + }*/ + + default boolean createIndex(String indexName){ + //TODO + return Boolean.TRUE; + } + + Boolean searchIndex(String index, String query, String dataContextVersion) throws Exception; +} diff --git a/dashboard-ingest/src/main/java/com/ingestpipeline/service/IngestService.java b/dashboard-ingest/src/main/java/com/ingestpipeline/service/IngestService.java new file mode 100644 index 000000000..1dc0a4355 --- /dev/null +++ b/dashboard-ingest/src/main/java/com/ingestpipeline/service/IngestService.java @@ -0,0 +1,14 @@ +package com.ingestpipeline.service; + +import org.springframework.stereotype.Service; + +import com.ingestpipeline.model.IncomingData; + +@Service +public interface IngestService { + + static final String TOPIC_CONTEXT_CONFIG = "TopicContextConfiguration.json"; + Boolean ingestToPipeline(Object incomingData); + IncomingData getContextForIncomingTopic(String topicName) ; + +} diff --git a/dashboard-ingest/src/main/java/com/ingestpipeline/service/IngestServiceImpl.java b/dashboard-ingest/src/main/java/com/ingestpipeline/service/IngestServiceImpl.java new file mode 100644 index 000000000..737d507ce --- /dev/null +++ b/dashboard-ingest/src/main/java/com/ingestpipeline/service/IngestServiceImpl.java @@ -0,0 +1,116 @@ +package com.ingestpipeline.service; + +import com.ingestpipeline.model.TopicContextConfig; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.ingestpipeline.model.ErrorWrapper; +import com.ingestpipeline.model.IncomingData; +import com.ingestpipeline.model.TopicContext; +import com.ingestpipeline.producer.IngestProducer; +import com.ingestpipeline.util.ApplicationProperties; +import com.ingestpipeline.util.ConfigLoader; +import com.ingestpipeline.util.Constants; + +import java.util.HashMap; +import java.util.Map; + +/** + * This is a Service Implementation for all the actions which are with respect to Elastic Search + * @author Darshan Nagesh + * + */ +@Service(Constants.Qualifiers.INGEST_SERVICE) +public class IngestServiceImpl implements IngestService { + + public static final Logger LOGGER = LoggerFactory.getLogger(IngestServiceImpl.class); + public static final String INTENT = "INGEST"; + + @Autowired + private IngestProducer ingestProducer; + + @Autowired + private ApplicationProperties applicationProperties; + + @Autowired + private ConfigLoader configLoader; + + @Autowired + private DigressService digressService; + + @Autowired + private ObjectMapper mapper; + + private Map topicContextMap = new HashMap<>(); + public void loadTopicsConfig(){ + TopicContextConfig topicContextConf = null; + try { + topicContextConf = mapper.readValue(configLoader.get(TOPIC_CONTEXT_CONFIG), TopicContextConfig.class); + for (TopicContext topicCxt : topicContextConf.getTopicContexts()){ + + topicContextMap.put(topicCxt.getTopic(), topicCxt); + } + LOGGER.info("topicContexts ## "+topicContextMap); + + } catch (Exception e) { + LOGGER.error("Encountered an error while reading Topic to Context Configuration" + e.getMessage()); + e.printStackTrace(); + } + } + + @Override + public Boolean ingestToPipeline(Object incomingData) { + LOGGER.info("Fetching the Incoming Data Config for the data received"); + String topic = ""; + String key = ""; + try { + if (applicationProperties.getPipelineRules().get(Constants.PipelineRules.VALIDATE_DATA)) { + topic = applicationProperties.getTransactionIngestTopic(); + key = applicationProperties.getTransactionIngestKey(); + } else if (applicationProperties.getPipelineRules().get(Constants.PipelineRules.TRANSFORM_DATA)) { + topic = applicationProperties.getTransactionValidationTopic(); + key = applicationProperties.getTransactionValidationKey(); + } else if (applicationProperties.getPipelineRules().get(Constants.PipelineRules.ENRICH_DATA)) { + topic = applicationProperties.getTransactionTransformationTopic(); + key = applicationProperties.getTransactionTransformationKey(); + } + } catch (Exception e) { + LOGGER.error("Encountered an Exception while Pushing the Data to pipeline on Ingest Service " + e.getMessage()); + ErrorWrapper errorWrapper = errorHandover(incomingData); + ingestProducer.pushToPipeline(errorWrapper, Constants.KafkaTopics.ERROR_INTENT, Constants.KafkaTopics.ERROR_INTENT); + } + ingestProducer.pushToPipeline(incomingData, topic, key); + return true; + } + + private ErrorWrapper errorHandover(Object incomingData) { + ErrorWrapper errorWrapper = new ErrorWrapper(); + errorWrapper.setErrorCode(INTENT); + errorWrapper.setErrorMessage(Constants.ErrorMessages.errorCodeMessageMap.get(INTENT)); + errorWrapper.setIncomingData(incomingData); + return errorWrapper; + } + + @Override + public IncomingData getContextForIncomingTopic(String topicName) { + ObjectMapper mapper = new ObjectMapper(); + TopicContext topicContext = null; + IncomingData incomingData = null; +/* try { + topicContext = mapper.readValue(configLoader.get(TOPIC_CONTEXT_CONFIG), TopicContext.class); + } catch (Exception e) { + LOGGER.error("Encountered an error while reading Topic to Context Configuration" + e.getMessage()); + }*/ + if(topicContextMap.size()>0 ) { + incomingData = new IncomingData(); + TopicContext topicCxt = topicContextMap.get(topicName); + incomingData.setDataContext(topicCxt.getDataContext()); + incomingData.setDataContextVersion(topicCxt.getDataContextVersion()); + LOGGER.info("###Context Set##"); + } + return incomingData; + } +} diff --git a/dashboard-ingest/src/main/java/com/ingestpipeline/service/RetryTemplate.java b/dashboard-ingest/src/main/java/com/ingestpipeline/service/RetryTemplate.java new file mode 100644 index 000000000..e42d532e2 --- /dev/null +++ b/dashboard-ingest/src/main/java/com/ingestpipeline/service/RetryTemplate.java @@ -0,0 +1,39 @@ +package com.ingestpipeline.service; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.http.HttpEntity; +import org.springframework.http.HttpMethod; +import org.springframework.http.ResponseEntity; +import org.springframework.retry.annotation.Backoff; +import org.springframework.retry.annotation.Retryable; +import org.springframework.stereotype.Component; +import org.springframework.web.client.ResourceAccessException; +import org.springframework.web.client.RestTemplate; + +import javax.naming.ServiceUnavailableException; + +/** + * Wraps rest template with retry + */ +@Component +public class RetryTemplate { + + @Autowired + private RestTemplate restTemplate; + + @Retryable(value = {RuntimeException.class, ResourceAccessException.class, ServiceUnavailableException.class}, + maxAttemptsExpression = "#{${service.retry.maxAttempts}}", + backoff = @Backoff(delayExpression = "#{${service.retry.backoff.delay}}")) + public ResponseEntity postForEntity(String url, Object request) { + return restTemplate.postForEntity(url, request, Object.class); + } + + @Retryable(value = {RuntimeException.class, ResourceAccessException.class, ServiceUnavailableException.class}, + maxAttemptsExpression = "#{${service.retry.maxAttempts}}", + backoff = @Backoff(delayExpression = "#{${service.retry.backoff.delay}}")) + public ResponseEntity getForEntity(String url, HttpEntity headerEntity) { + return restTemplate.exchange(url, HttpMethod.GET, headerEntity, Object.class); + } + +} + diff --git a/dashboard-ingest/src/main/java/com/ingestpipeline/service/TransformService.java b/dashboard-ingest/src/main/java/com/ingestpipeline/service/TransformService.java new file mode 100644 index 000000000..84781d806 --- /dev/null +++ b/dashboard-ingest/src/main/java/com/ingestpipeline/service/TransformService.java @@ -0,0 +1,14 @@ +package com.ingestpipeline.service; + +import org.springframework.stereotype.Service; + +import com.ingestpipeline.model.IncomingData; + +import java.util.Map; + +@Service +public interface TransformService { + + Boolean transformData(Map incomingData); + +} diff --git a/dashboard-ingest/src/main/java/com/ingestpipeline/service/TransformServiceImpl.java b/dashboard-ingest/src/main/java/com/ingestpipeline/service/TransformServiceImpl.java new file mode 100644 index 000000000..d27f20989 --- /dev/null +++ b/dashboard-ingest/src/main/java/com/ingestpipeline/service/TransformServiceImpl.java @@ -0,0 +1,117 @@ +package com.ingestpipeline.service; + +import java.util.List; +import java.util.Map; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import com.bazaarvoice.jolt.Chainr; +import com.bazaarvoice.jolt.JsonUtils; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.gson.Gson; +import com.ingestpipeline.model.DigressionPoint; +import com.ingestpipeline.util.ConfigLoader; +import com.ingestpipeline.util.Constants; + +/** + * This is a Service Implementation for all the actions which are with respect to Elastic Search + * @author Darshan Nagesh + * + */ +@Service(Constants.Qualifiers.TRANSFORM_SERVICE) +public class TransformServiceImpl implements TransformService { + + public static final Logger LOGGER = LoggerFactory.getLogger(TransformServiceImpl.class); + private static final String SEPARATOR = "_"; + private static final String JSON_EXTENSION = ".json"; + private static final String OBJECTIVE = "transform"; + private static final String CONFIGROOT = "config/"; + + @Autowired + private ConfigLoader configLoader; + + @Override + public Boolean transformData(Map incomingData) { + String dataContext = incomingData.get(Constants.DATA_CONTEXT).toString(); + String dataContextVersion = incomingData.get(Constants.DATA_CONTEXT_VERSION).toString(); + ObjectMapper mapper = new ObjectMapper(); + List chainrSpecJSON = null ; + try { + chainrSpecJSON = mapper.readValue(configLoader.get(OBJECTIVE.concat(SEPARATOR).concat(dataContext).concat(SEPARATOR).concat(dataContextVersion).concat(JSON_EXTENSION)), List.class); + + } catch (Exception e) { + LOGGER.error("Encountered an error : " + e.getMessage()); + } + String sourceUrl = CONFIGROOT.concat(OBJECTIVE.concat(SEPARATOR).concat(dataContext).concat(SEPARATOR).concat(dataContextVersion).concat(JSON_EXTENSION)); + chainrSpecJSON = JsonUtils.jsonToList(this.getClass().getClassLoader().getResourceAsStream(sourceUrl)); + + /*Map deepRouteSpecSize = new HashMap<>(); + + Object dataObject = incomingData.get(Constants.DATA_OBJECT); + String digressConfig = configLoader.get(Constants.ConfigurationFileNames.DIGRESSION_POINTS); + Gson gson = new Gson(); + JsonNode mainNode = null; + try { + mainNode = mapper.readTree(gson.toJson(dataObject)); + } catch (Exception ex) { + LOGGER.error("Encountered an error while converting the data to JSON Node : " + ex.getMessage()); + } + ObjectNode objectNode = mainNode.deepCopy(); + DigressionPointConfig config = gson.fromJson(digressConfig, DigressionPointConfig.class); + if (config != null && config.getDigressionPoints() != null && config.getDigressionPoints().size() > 0) { + for (DigressionPoint point : config.getDigressionPoints()) { + if (incomingData.get(Constants.DATA_CONTEXT).equals(point.getDataContext())) { + List paths = point.getPaths(); + for(String path : paths) { + if(objectNode.findValue(path).isArray()) { + ArrayNode objectAtPathArray = (ArrayNode) objectNode.findValue(path); + int size = objectAtPathArray.size(); + deepRouteSpecSize.put(path, size); + } + } + + } + } + } + + + for(int i=0; i< chainrSpecJSON.size() ; i++) { + LinkedHashMap chainSpec = (LinkedHashMap) chainrSpecJSON.get(i); + for(Map.Entry entry : deepRouteSpecSize.entrySet()) { + String key = entry.getKey(); + int iterationsForKey = entry.getValue(); + for() + } + chainSpec.toString(); + }*/ + + + Chainr chainr = Chainr.fromSpec( chainrSpecJSON ); + Object inputJSON = incomingData.get(Constants.DATA_OBJECT); + try { + Object transformedOutput = chainr.transform( inputJSON ); + incomingData.put(Constants.DATA_OBJECT , transformedOutput); + return Boolean.TRUE; + } catch (Exception e) { + LOGGER.error("Encountered an error while tranforming the JSON : " + e.getMessage()); + return Boolean.FALSE; + } + } + + public Boolean digressData(Object dataObject, DigressionPoint digressionPoint) { + ObjectMapper mapper = new ObjectMapper(); + Gson gson = new Gson(); + JsonNode mainNode = null; + try { + mainNode = mapper.readTree(gson.toJson(dataObject)); + } catch (Exception ex) { + LOGGER.error("Encountered an error while converting the data to JSON Node : " + ex.getMessage()); + } + return null; + } +} + \ No newline at end of file diff --git a/dashboard-ingest/src/main/java/com/ingestpipeline/service/ValidationService.java b/dashboard-ingest/src/main/java/com/ingestpipeline/service/ValidationService.java new file mode 100644 index 000000000..d06e5cde2 --- /dev/null +++ b/dashboard-ingest/src/main/java/com/ingestpipeline/service/ValidationService.java @@ -0,0 +1,14 @@ +package com.ingestpipeline.service; + +import org.springframework.stereotype.Service; + +import com.ingestpipeline.model.IncomingData; + +import java.util.Map; + +@Service +public interface ValidationService { + + Boolean validateData(Map incomingData); + +} diff --git a/dashboard-ingest/src/main/java/com/ingestpipeline/service/ValidationServiceImpl.java b/dashboard-ingest/src/main/java/com/ingestpipeline/service/ValidationServiceImpl.java new file mode 100644 index 000000000..e211230ea --- /dev/null +++ b/dashboard-ingest/src/main/java/com/ingestpipeline/service/ValidationServiceImpl.java @@ -0,0 +1,73 @@ +package com.ingestpipeline.service; + +import java.util.HashMap; +import java.util.Map; + +import org.everit.json.schema.Schema; +import org.everit.json.schema.ValidationException; +import org.everit.json.schema.loader.SchemaLoader; +import org.json.JSONObject; +import org.json.JSONTokener; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Service; + +import com.google.gson.GsonBuilder; +import com.ingestpipeline.util.Constants; + +/** + * This is a Service Implementation for all the actions which are with respect to Elastic Search + * @author Darshan Nagesh + * + */ +@Service(Constants.Qualifiers.VALIDATOR_SERVICE) +public class ValidationServiceImpl implements ValidationService { + + public static final Logger LOGGER = LoggerFactory.getLogger(ValidationServiceImpl.class); + private static final String OBJECTIVE = "validator"; + private static final String SEPARATOR = "_"; + private static final String JSON_EXTENSION = ".json"; + private static final String CONFIGROOT = "config/"; + private static Map schemaCache = new HashMap<>(); + + @Override + public Boolean validateData(Map incomingData) { + + //JSONObject jsonSubject = new JSONObject(new GsonBuilder().create().toJson(incomingData.getDataObject())); + JSONObject jsonSubject = new JSONObject(new GsonBuilder().create().toJson(incomingData.get("dataObject"))); + + try { + //getSchema(OBJECTIVE.concat(SEPARATOR).concat(incomingData.getDataContext()).concat(SEPARATOR).concat(incomingData.getDataContextVersion()) + getSchema(OBJECTIVE.concat(SEPARATOR).concat(incomingData.get("dataContext").toString()).concat(SEPARATOR).concat(incomingData.get("dataContextVersion").toString()) + .concat(JSON_EXTENSION)).validate(jsonSubject); + } catch (ValidationException ve) { + LOGGER.info("Validation Exception : " + ve.getMessage()); + return Boolean.FALSE; + } catch (Exception e) { + LOGGER.info("Validation Exception : " + e); + return Boolean.FALSE; + } + return Boolean.TRUE; + } + + private Schema getSchema(String location) { + Schema schema = schemaCache.get(location); + if (schema == null) { + try { + schema = loadSchema(location); + schemaCache.put(location, schema); + } catch(Exception e) { + LOGGER.error("Encountered an Exception while loading Schema : " + e.getMessage()); + } + } + return schema; + } + + private Schema loadSchema(String location) { + JSONObject jsonSchema = new JSONObject( + new JSONTokener(ClassLoader.getSystemClassLoader().getResourceAsStream(CONFIGROOT + location))); + return SchemaLoader.load(jsonSchema); + } + +} + \ No newline at end of file diff --git a/dashboard-ingest/src/main/java/com/ingestpipeline/util/ApplicationProperties.java b/dashboard-ingest/src/main/java/com/ingestpipeline/util/ApplicationProperties.java new file mode 100644 index 000000000..bb0b96def --- /dev/null +++ b/dashboard-ingest/src/main/java/com/ingestpipeline/util/ApplicationProperties.java @@ -0,0 +1,162 @@ +package com.ingestpipeline.util; + +import java.util.Map; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.PropertySource; +import org.springframework.core.annotation.Order; +import org.springframework.core.env.Environment; + +@Configuration +@PropertySource(value = { "classpath:config/config.properties" }, ignoreResourceNotFound = true) +@Order(0) +public class ApplicationProperties { + + @Value("${kafka.transaction.ingest.topic}") + private String transactionIngestTopic; + + @Value("${kafka.transaction.ingest.key}") + private String transactionIngestKey; + + @Value("${kafka.transaction.validation.topic}") + private String transactionValidationTopic; + + @Value("${kafka.transaction.validation.key}") + private String transactionValidationKey; + + @Value("${kafka.transaction.transformation.topic}") + private String transactionTransformationTopic; + + @Value("${kafka.transaction.transformation.key}") + private String transactionTransformationKey; + + @Value("${kafka.transaction.enrichment.topic}") + private String transactionEnrichmentTopic; + + @Value("${kafka.transaction.enrichment.key}") + private String transactionEnrichmentKey; + + @Value("${schema.transaction.validation}") + private String transactionValidationSchema; + + @Value("${schema.transaction.transformation}") + private String transactionTransformSchema; + + @Value("${schema.transaction.enrichment}") + private String transactionEnrichmentSchema; + + @Value("${transformation.config.location}") + private static String transformationConfigLocations; + + @Value("#{${pipelinerules}}") + private Map pipelineRules; + + @Autowired + private Environment environment; + + public static String getTransformationConfigLocations() { + return transformationConfigLocations; + } + + public static void setTransformationConfigLocations(String transformationConfigLocations) { + ApplicationProperties.transformationConfigLocations = transformationConfigLocations; + } + + public String getTransactionValidationSchema() { + return transactionValidationSchema; + } + + public void setTransactionValidationSchema(String transactionValidationSchema) { + this.transactionValidationSchema = transactionValidationSchema; + } + + public String getTransactionTransformSchema() { + return transactionTransformSchema; + } + + public void setTransactionTransformSchema(String transactionTransformSchema) { + this.transactionTransformSchema = transactionTransformSchema; + } + + public String getTransactionEnrichmentSchema() { + return transactionEnrichmentSchema; + } + + public void setTransactionEnrichmentSchema(String transactionEnrichmentSchema) { + this.transactionEnrichmentSchema = transactionEnrichmentSchema; + } + + public Map getPipelineRules() { + return pipelineRules; + } + + public void setPipelineRules(Map pipelineRules) { + this.pipelineRules = pipelineRules; + } + + public String getTransactionIngestTopic() { + return transactionIngestTopic; + } + + public void setTransactionIngestTopic(String transactionIngestTopic) { + this.transactionIngestTopic = transactionIngestTopic; + } + + public String getTransactionIngestKey() { + return transactionIngestKey; + } + + public void setTransactionIngestKey(String transactionIngestKey) { + this.transactionIngestKey = transactionIngestKey; + } + + public String getTransactionValidationTopic() { + return transactionValidationTopic; + } + + public void setTransactionValidationTopic(String transactionValidationTopic) { + this.transactionValidationTopic = transactionValidationTopic; + } + + public String getTransactionValidationKey() { + return transactionValidationKey; + } + + public void setTransactionValidationKey(String transactionValidationKey) { + this.transactionValidationKey = transactionValidationKey; + } + + public String getTransactionTransformationTopic() { + return transactionTransformationTopic; + } + + public void setTransactionTransformationTopic(String transactionTransformationTopic) { + this.transactionTransformationTopic = transactionTransformationTopic; + } + + public String getTransactionTransformationKey() { + return transactionTransformationKey; + } + + public void setTransactionTransformationKey(String transactionTransformationKey) { + this.transactionTransformationKey = transactionTransformationKey; + } + + public String getTransactionEnrichmentTopic() { + return transactionEnrichmentTopic; + } + + public void setTransactionEnrichmentTopic(String transactionEnrichmentTopic) { + this.transactionEnrichmentTopic = transactionEnrichmentTopic; + } + + public String getTransactionEnrichmentKey() { + return transactionEnrichmentKey; + } + + public void setTransactionEnrichmentKey(String transactionEnrichmentKey) { + this.transactionEnrichmentKey = transactionEnrichmentKey; + } +} diff --git a/dashboard-ingest/src/main/java/com/ingestpipeline/util/ConfigLoader.java b/dashboard-ingest/src/main/java/com/ingestpipeline/util/ConfigLoader.java new file mode 100644 index 000000000..09295d534 --- /dev/null +++ b/dashboard-ingest/src/main/java/com/ingestpipeline/util/ConfigLoader.java @@ -0,0 +1,86 @@ +package com.ingestpipeline.util; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.Charset; +import java.util.HashMap; +import java.util.Map; + +import org.apache.commons.io.IOUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.core.io.Resource; +import org.springframework.core.io.ResourceLoader; +import org.springframework.core.io.support.ResourcePatternUtils; +import org.springframework.stereotype.Component; + +@Component("configLoader") +public class ConfigLoader { + + private static Logger logger = LoggerFactory.getLogger(ConfigLoader.class); + private Map nameContentMap = new HashMap<>(); + @Autowired + private ResourceLoader resourceLoader; + + @Value("${config.schema.paths}") + private String RESOURCE_LOCATION; + + /** + * Loads config resources + * @throws Exception + */ + public void loadResources() throws Exception { + logger.info("RESOURCE_LOCATION:: "+RESOURCE_LOCATION); + Resource[] resources = getResources(RESOURCE_LOCATION); + + for (Resource resource : resources) { + String jsonContent = getContent(resource); + nameContentMap.put(resource.getFilename(), jsonContent); + } + logger.info("Number of resources loaded " + nameContentMap.size()); + + } + + /** + * To fetch a particular string content for a give resource/file name + * @param name + * @return + */ + public String get(String name) { + return nameContentMap.get(name); + } + + /** + * Loads all the resources/files with a given pattern *.json + * @param pattern path with *json + * @return + * @throws IOException + */ + private Resource[] getResources(String pattern) throws IOException { + Resource[] resources = ResourcePatternUtils.getResourcePatternResolver(resourceLoader).getResources(pattern); + return resources; + } + + /** + * Returns a content of resource + * + * @param resource + * @return + */ + private String getContent(Resource resource) { + String content = null; + try { + InputStream is = resource.getInputStream(); + byte[] encoded = IOUtils.toByteArray(is); + content = new String(encoded, Charset.forName("UTF-8")); + + } catch (IOException e) { + logger.error("Cannot load resource " + resource.getFilename()); + + } + return content; + } + +} diff --git a/dashboard-ingest/src/main/java/com/ingestpipeline/util/Constants.java b/dashboard-ingest/src/main/java/com/ingestpipeline/util/Constants.java new file mode 100644 index 000000000..ec1d80030 --- /dev/null +++ b/dashboard-ingest/src/main/java/com/ingestpipeline/util/Constants.java @@ -0,0 +1,125 @@ +package com.ingestpipeline.util; + +import java.util.HashMap; +import java.util.Map; + +/** + * Constants which are with respect to the Ingest App + * + * @author Darshan Nagesh + * + */ + +public interface Constants { + public interface Paths { + final String ELASTIC_PUSH_CONTROLLER_PATH = "/ingest"; + final String SAVE = "/save"; + final String UPLOAD = "/upload"; + final String Targets = "/targets"; + final String Collections = "/getCollections"; + final String ES_INDEX = "/migrate/{indexName}/{version}"; + } + + public interface Qualifiers { + final String INGEST_SERVICE = "ingestService"; + final String VALIDATOR_SERVICE = "validatorService"; + final String TRANSFORM_SERVICE = "transformService"; + final String ENRICHMENT_SERVICE = "enrichmentService"; + final String TRANSFORM_COLLECTION_SERVICE = "transformCollectionService"; + final String DIGRESS_SERVICE = "digressService"; + } + + public static String SUCCESS = "success"; + public static int UNAUTHORIZED_ID = 401; + public static int SUCCESS_ID = 200; + public static int FAILURE_ID = 320; + public static String UNAUTHORIZED = "Invalid credentials. Please try again."; + public static String PROCESS_FAIL = "Process failed, Please try again."; + public static String DATE_FORMAT = "yyyy.MM.dd G 'at' HH:mm:ss z"; + public static String INDIAN_TIMEZONE = "IST"; + + public static String ALLOWED_METHODS_GET = "GET"; + public static String ALLOWED_METHODS_POST = "POST"; + public static String MDMS_MCOLLECT_SEARCH = "{\"MdmsCriteria\":{\"tenantId\":\"TENANTID_PLACEHOLDER\",\"moduleDetails\":[{\"moduleName\":\"BillingService\",\"masterDetails\":[{\"name\":\"BusinessService\",\"filter\":\"[?(@.type=='Adhoc')]\"}]}]},\"RequestInfo\":{}}"; + public static String TENANTID_PLACEHOLDER = "TENANTID_PLACEHOLDER"; + public static String CONTENT_TYPE = "content-type"; + public static String JSON = "application/json;charset=UTF-8"; + public static String MDMS_URL = "http://egov-mdms-service.egov:8080/egov-mdms-service/v1/_search"; + public static String MDMS_RES = "MdmsRes"; + public static String BUSINESS_SERVICE="BusinessService"; + public static String BILLING_SERVICE = "BillingService"; + public static String CATEGORY_CODE= "code"; + + public interface KafkaTopics { + public static final String INGEST_DATA = "ingestData"; + public static final String VALID_DATA = "validData"; + public static final String TRANSFORMED_DATA = "transformedData"; + public static final String ERROR_INTENT = "DataError"; + public static final String TOPIC_ONE = "topicOne"; + public static final String TOPIC_TWO = "topicTwo"; + public static final String TOPIC_THREE = "topicThree"; + public static final String TOPIC_FOUR = "topicFour"; + } + + public interface BeanContainerFactory { + public static final String INCOMING_KAFKA_LISTENER = "incomingKafkaListenerContainerFactory"; + } + + public interface DomainConfigurations { + public static final String COLLECTION_DOMAIN_CONFIG = "DomainConfig"; + public static final String ENHANCE_DOMAIN_CONFIG = "EnhanceDomainConfig"; + } + + public interface PipelineRules { + public static final String VALIDATE_DATA = "VALIDATE"; + public static final String TRANSFORM_DATA = "TRANSFORM"; + public static final String ENRICH_DATA = "ENRICH"; + } + + public interface ScrollSearch { + public static final String SCROLL_ID = "scrollId"; + public static final String SEARCH_PATH = "searchPath"; + public static final String QUERY = "query"; + public static final String SCROLL_ID_PARAMS = "_scroll_id"; + public static final String SCROLL_SEARCH_DEFAULT_QUERY = "{\"scroll\":\"1m\",\"scroll_id\":"; + } + + public interface DataContexts { + public static final String CONTEXT = "context"; + public static final String COLLECTION = "collection"; + public static final String BILLING = "billing" ; + public static final String PAYMENT = "payment" ; + } + + public static String DATA_CONTEXT = "dataContext"; + public static String DATA_CONTEXT_VERSION = "dataContextVersion"; + public static String DATA_OBJECT = "dataObject"; + + public static String ERROR_IN_PIPEINE = "errorPipeline"; + + public static int HEADER_ROW = 1; + public static String MUNICIPAL_CORPORATIONS = "Municipal Corporations"; + public static String ES_INDEX_COLLECTION = "collectionsindex-v1"; + public static String ES_INDEX_BILLING = "billingservice"; + public static String ES_INDEX_PAYMENT = "paymentsindex-v1"; + + public interface ErrorMessages { + Map errorCodeMessageMap = new HashMap() {{ + put("INGEST","Error finding Context for the Topic"); + put("VALIDATE","Error while validating the Document"); + put("TRANSFORM","Error while transforming the Document"); + put("ENRICH","Error while enriching the Document"); + }}; + } + + public static interface ConfigurationFileNames { + public static final String DIGRESSION_POINTS = "DigressionPoints.json"; + } + public static interface TransformationType { + public static final String COLLECTION = "collection"; + + } + public static final String IDENTIFIER = "identifier"; + public static final String TRANSACTION_ID = "transactionId"; + public static final String ID = "id"; +} diff --git a/dashboard-ingest/src/main/java/com/ingestpipeline/util/CustomErrorType.java b/dashboard-ingest/src/main/java/com/ingestpipeline/util/CustomErrorType.java new file mode 100644 index 000000000..687121237 --- /dev/null +++ b/dashboard-ingest/src/main/java/com/ingestpipeline/util/CustomErrorType.java @@ -0,0 +1,16 @@ +package com.ingestpipeline.util; + + +public class CustomErrorType { + + private String errorMessage; + + public CustomErrorType(String errorMessage){ + this.errorMessage = errorMessage; + } + + public String getErrorMessage() { + return errorMessage; + } + +} diff --git a/dashboard-ingest/src/main/java/com/ingestpipeline/util/JSONUtil.java b/dashboard-ingest/src/main/java/com/ingestpipeline/util/JSONUtil.java new file mode 100644 index 000000000..55b120f20 --- /dev/null +++ b/dashboard-ingest/src/main/java/com/ingestpipeline/util/JSONUtil.java @@ -0,0 +1,125 @@ +package com.ingestpipeline.util; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.google.gson.Gson; +import com.ingestpipeline.repository.ElasticSearchRepository; + +import org.json.JSONArray; +import org.json.JSONObject; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.http.ResponseEntity; +import org.springframework.stereotype.Service; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +@Service +public class JSONUtil { + @Autowired + public ObjectMapper mapper; + @Autowired + public Gson gson; + @Autowired + private ElasticSearchRepository elasticRepository; + + /** + * Field value to replace by new text. Replace node by given text to Parent's + * hierarchy. Field will not be added if not found existing already + * + * @param parent + * @param fieldName + * @param newValue + */ + public static void replaceFieldValue(ObjectNode parent, String fieldName, String newValue) { + if (parent.has(fieldName)) { + parent.put(fieldName, newValue); + } + parent.fields().forEachRemaining(entry -> { + JsonNode entryValue = entry.getValue(); + if (entryValue.isArray()) { + for (int i = 0; i < entryValue.size(); i++) { + if (entry.getValue().get(i).isObject()) + replaceFieldValue((ObjectNode) entry.getValue().get(i), fieldName, newValue); + } + } else if (entryValue.isObject()) { + replaceFieldValue((ObjectNode) entry.getValue(), fieldName, newValue); + } + }); + } + + /** + * Finds the value for a given key from the tree + * @param parent the tree + * @param key the field name + * @param value the value for the given field + */ + public static void findValue(ObjectNode parent, String key, StringBuffer value) { + parent.fields().forEachRemaining(entry -> { + if(entry.getKey().equalsIgnoreCase(key)){ + value.append(entry.getValue().asText()); + } else { + JsonNode entryValue = entry.getValue(); + if(entryValue.isArray()){ + for (int i=0; i < entryValue.size(); i++){ + if (entry.getValue().get(i).isObject()) + findValue((ObjectNode) entry.getValue().get(i), key, value); + } + } else if (entryValue.isObject()){ + findValue((ObjectNode) entry.getValue(), key, value); + } + } + }); + } + + + /** + * @return + */ + public static String getJsonString(ObjectMapper objectMapper,Object object) throws JsonProcessingException { + if(objectMapper != null){ + return objectMapper.writeValueAsString(object); + } + return null; + } + + public ObjectMapper getMapper() { + return mapper; + } + + public void setObjectMapper(ObjectMapper objectMapper){ + mapper=objectMapper; + } + + public Gson getGson() { + return gson; + } + + public void setGson(Gson gsonn) + { + gson = gsonn; + } + + public List fetchMCollectCategories (String tenantId) { + + String mdmsRequestString = Constants.MDMS_MCOLLECT_SEARCH.replace(Constants.TENANTID_PLACEHOLDER, tenantId); + Object mdmsRequestObject = mdmsRequestString; + + ResponseEntity result = elasticRepository.fetchMDMSResponse(mdmsRequestObject); + + Map body = result.getBody(); + JSONObject jsonObj = new JSONObject(body); + JSONArray jsonArray = jsonObj.getJSONObject(Constants.MDMS_RES).getJSONObject(Constants.BILLING_SERVICE).getJSONArray(Constants.BUSINESS_SERVICE); + List categoriesList = new ArrayList(); + + for(int i = 0 ; i < jsonArray.length() ; i++){ + categoriesList.add(jsonArray.getJSONObject(i).getString(Constants.CATEGORY_CODE)); + } + + return categoriesList; + + } +} diff --git a/dashboard-ingest/src/main/java/com/ingestpipeline/util/JSONUtils.java b/dashboard-ingest/src/main/java/com/ingestpipeline/util/JSONUtils.java new file mode 100644 index 000000000..b317e8be0 --- /dev/null +++ b/dashboard-ingest/src/main/java/com/ingestpipeline/util/JSONUtils.java @@ -0,0 +1,16 @@ +package com.ingestpipeline.util; + +import org.json.JSONObject; +import com.fasterxml.jackson.annotation.JsonPropertyOrder; + +@JsonPropertyOrder({ "dataContext", "dataContextVersion", "dataObject"}) +public class JSONUtils { + public JSONObject JSONWrapper(JSONObject json) { + JSONObject obj = new JSONObject(); + obj.put("dataContext", "target"); + obj.put("dataContextVersion", "v1"); + obj.put("dataObject", json); + return obj; + } + +} diff --git a/dashboard-ingest/src/main/java/com/ingestpipeline/util/JsonKey.java b/dashboard-ingest/src/main/java/com/ingestpipeline/util/JsonKey.java new file mode 100644 index 000000000..27dcd8a32 --- /dev/null +++ b/dashboard-ingest/src/main/java/com/ingestpipeline/util/JsonKey.java @@ -0,0 +1,20 @@ +/** + * + */ +package com.ingestpipeline.util; + +/** + * @author Abhishek + * + */ +public class JsonKey { + + public static final String STATUS_CODE = "statusCode"; + public static final String STATUS = "statusInfo"; + public static final String STATUS_MESSAGE = "statusMessage"; + public static final String ERROR_MESSAGE = "errorMessage"; + + public static final String RESPONSE_DATA = "responseData"; + + +} diff --git a/dashboard-ingest/src/main/java/com/ingestpipeline/util/MockMultipartFile.java b/dashboard-ingest/src/main/java/com/ingestpipeline/util/MockMultipartFile.java new file mode 100644 index 000000000..f678698ef --- /dev/null +++ b/dashboard-ingest/src/main/java/com/ingestpipeline/util/MockMultipartFile.java @@ -0,0 +1,77 @@ +package com.ingestpipeline.util; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; + +import org.springframework.web.multipart.MultipartFile; + +public class MockMultipartFile implements MultipartFile { + + private String name; + private String originalFileName; + private String contentType; + private byte[] content; + + public MockMultipartFile(String name, String originalFileName, String contentType, byte[] content) { + super(); + this.name = name; + this.originalFileName = originalFileName; + this.contentType = contentType; + this.content = content; + } + + @Override + public String getName() { + // TODO Auto-generated method stub + return null; + } + + @Override + public String getOriginalFilename() { + // TODO Auto-generated method stub + return null; + } + + @Override + public String getContentType() { + // TODO Auto-generated method stub + return null; + } + + @Override + public boolean isEmpty() { + // TODO Auto-generated method stub + return false; + } + + @Override + public long getSize() { + // TODO Auto-generated method stub + return 0; + } + + @Override + public byte[] getBytes() throws IOException { + // TODO Auto-generated method stub + return null; + } + + @Override + public InputStream getInputStream() throws IOException { + // TODO Auto-generated method stub + return null; + } + + @Override + public void transferTo(File dest) throws IOException, IllegalStateException { + // TODO Auto-generated method stub + + } + + + + + + +} diff --git a/dashboard-ingest/src/main/java/com/ingestpipeline/util/ReadUtil.java b/dashboard-ingest/src/main/java/com/ingestpipeline/util/ReadUtil.java new file mode 100644 index 000000000..1fa86e44f --- /dev/null +++ b/dashboard-ingest/src/main/java/com/ingestpipeline/util/ReadUtil.java @@ -0,0 +1,347 @@ +package com.ingestpipeline.util; + +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileOutputStream; +import java.io.FileWriter; +import java.io.IOException; +import java.io.OutputStream; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.text.DateFormat; +import java.text.DecimalFormat; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Date; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.ListIterator; +import java.util.Map; +import java.util.Map.Entry; +import java.util.NavigableMap; +import java.util.Scanner; +import java.util.TreeMap; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.apache.commons.fileupload.disk.DiskFileItem; +import org.apache.commons.io.FilenameUtils; +import org.apache.commons.io.IOUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.poi.ss.usermodel.Cell; +import org.apache.poi.ss.usermodel.CellType; +import org.apache.poi.ss.usermodel.DataFormatter; +import org.apache.poi.ss.usermodel.DateUtil; +import org.apache.poi.ss.usermodel.RichTextString; +import org.apache.poi.ss.usermodel.Row; +import org.apache.poi.ss.usermodel.Sheet; +import org.apache.poi.ss.usermodel.Workbook; +import org.apache.poi.ss.usermodel.WorkbookFactory; +import org.apache.poi.ss.util.CellRangeAddress; +import org.apache.poi.xssf.usermodel.XSSFWorkbook; +import org.elasticsearch.search.aggregations.support.ValuesSource.Numeric; +import org.json.JSONArray; +import org.json.JSONObject; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; +import org.springframework.web.multipart.MultipartFile; +import org.springframework.web.multipart.commons.CommonsMultipartFile; + +import com.ingestpipeline.controller.RestApiController; +import com.ingestpipeline.service.IngestService; + +@Component("readUtil") +public class ReadUtil { + + public static final Logger LOGGER = LoggerFactory.getLogger(ReadUtil.class); + private static String UPLOADED_FOLDER = ""; + public static Path path; + public static File uploadFile = new File( + System.getProperty("user.dir") + System.getProperty("file.separator") + "uploads"); + + public static JSONArray getFiletoDirectory(MultipartFile file) throws Exception { + byte[] bytes = file.getBytes(); + if (!uploadFile.exists()) { + uploadFile.mkdir(); + } + UPLOADED_FOLDER = uploadFile.toString(); + path = Paths.get(UPLOADED_FOLDER + System.getProperty("file.separator") + file.getOriginalFilename()); + Files.write(path, bytes); + JSONArray fileIntoJsonArray = readFilefromDirectory(); + String filename = null; + Path p = Paths.get(file.getOriginalFilename()); + if (p.toString().indexOf(".") > 0) { + filename = p.toString().substring(0, p.toString().lastIndexOf(".")); + } + String jsonArrayFileName = filename + ".json"; + writeJsonArrayToFile(fileIntoJsonArray, jsonArrayFileName); + return fileIntoJsonArray; + } + + private static JSONArray readFilefromDirectory() throws Exception { + String workbookSheetName = null; + int workbookSheetIndex = -1; + String getFileExtension = FilenameUtils.getExtension(path.toString()); + Workbook workbook = null; + if (getFileExtension.endsWith("xlsx")) { + workbook = new XSSFWorkbook(); + workbook = WorkbookFactory.create(new File(path.toString())); + } else { + throw new Exception("invalid file, should be xlsx"); + } + + JSONArray workbookToJsonArray = new JSONArray(); + for (int i = 0; i < workbook.getNumberOfSheets(); i++) { + Sheet sheet = workbook.getSheetAt(i); + workbookSheetName = sheet.getSheetName(); + workbookSheetIndex = workbook.getSheetIndex(workbookSheetName); + if (workbookSheetName != null && workbookSheetName.length() > 0) { + workbookToJsonArray = workbookToJsonArray.put(getSheetToJsonObject(workbook, sheet)); + } + } + + JSONArray bookInJsonArray = new JSONArray(); + for (int i = 0; i < workbookToJsonArray.length(); i++) { + JSONArray sheetInJsonArray = new JSONArray(); + sheetInJsonArray = (JSONArray) workbookToJsonArray.get(i); + Iterator itr = sheetInJsonArray.iterator(); + while (itr.hasNext()) { + JSONObject obj = (JSONObject) itr.next(); + bookInJsonArray.put(obj); + } + } + return bookInJsonArray; + } + + private static JSONArray getSheetToJsonObject(Workbook workbook, Sheet sheet) { + String workbookSheetName = sheet.getSheetName(); + int workbookSheetIndex = workbook.getSheetIndex(workbookSheetName); + Object financialYear = null; + int firstRowNum = sheet.getFirstRowNum(), lastRowNum = sheet.getLastRowNum(), + ulbFirstRowNumber = Constants.HEADER_ROW + 2, ulbDestRowNumber = -1; + Row row = null; + int rowFirstCellNum = -1, rowLastCellNum = -1; + List> rowRecordList = new LinkedList>(); + Map> rowRecordMap = new LinkedHashMap>(); + Map> municipalCity = new LinkedHashMap>(); + + if (lastRowNum > 0) { + for (int i = firstRowNum; i <= lastRowNum; i++) { + row = sheet.getRow(i); + if (row != null) { + rowFirstCellNum = row.getFirstCellNum(); + rowLastCellNum = sheet.getRow(i).getLastCellNum(); + List rowRecord = new LinkedList(); + Map> singleRowDataMap = new LinkedHashMap>(); + for (int j = rowFirstCellNum; j < rowLastCellNum; j++) { + Cell cell = row.getCell(j); + if (isMergedRegion(workbook, sheet, i, j)) { + sheet = getSheetMergerCellRegion(workbook, sheet, i, j); + } + singleRowDataMap.put(i, getRowRecord(cell, rowRecord)); + } + + if (!rowRecord.isEmpty()) { + rowRecordList.add(rowRecord); + // rowRecordMap.put(i, rowRecord); + rowRecordMap.putAll(singleRowDataMap); + + } + + if (rowRecordList.size() == 1) { + financialYear = rowRecordList.get(Constants.HEADER_ROW - 1).get(3).toString().trim(); + } + } + } + } + + List sheetHeaderList = new LinkedList(); + List customHeaderList = new LinkedList(); + Map customHeaderMap = new LinkedHashMap(); + JSONArray getMunicipalCityToJsonArray = new JSONArray(); + Map> lastRowRecord = new LinkedHashMap>(); + customHeaderList.add("Sheet Name"); + customHeaderList.add("Financial Year"); + customHeaderList.add("Timestamp"); + // customHeaderList.add("Municipal Corporations"); + customHeaderMap.put(customHeaderList.get(0), workbookSheetName); + customHeaderMap.put(customHeaderList.get(1), financialYear); + DateFormat df = new SimpleDateFormat("dd/MM/yy HH:mm:S"); + Date dateobj = new Date(); + customHeaderMap.put(customHeaderList.get(2), df.format(dateobj)); + JSONObject customHeaderJsonObject = new JSONObject(); + JSONArray municipalCitiesIntoJsonArray = new JSONArray(); + for (Map.Entry> itrRowRecordMap : rowRecordMap.entrySet()) { + if (itrRowRecordMap.getKey() == Constants.HEADER_ROW) { + sheetHeaderList.addAll(itrRowRecordMap.getValue()); + } + if (itrRowRecordMap.getKey() >= ulbFirstRowNumber) { + JSONObject municipalCitiesIntoJsonObject = new JSONObject(); + Map mc = new LinkedHashMap(); + municipalCity.put(itrRowRecordMap.getKey(), itrRowRecordMap.getValue()); + for (Map.Entry itrCustomHeaderMap : customHeaderMap.entrySet()) { + municipalCitiesIntoJsonObject.accumulate(itrCustomHeaderMap.getKey().toString(), + itrCustomHeaderMap.getValue()); + } + + for (int i = 0; i < sheetHeaderList.size(); i++) { + if ((!sheetHeaderList.get(i).toString().contentEquals("-")) + && (!itrRowRecordMap.getValue().get(i).toString().contentEquals("-"))) { + + municipalCitiesIntoJsonObject.put(sheetHeaderList.get(i).toString(), + itrRowRecordMap.getValue().get(i)); + } + } + + municipalCitiesIntoJsonArray.put(municipalCitiesIntoJsonObject); + } + } + return municipalCitiesIntoJsonArray; + } + + private static List getRowRecord(Cell cell, List rowRecord) { + switch (cell.getCellType()) { + case STRING: + String str = cell.getRichStringCellValue().getString(); + str = str.replaceAll("(\\n\\s)+|(\\r\\n\\s)+|(\\r\\s)+|(\\r)+|(\\n)+|(\\s)+", " "); + rowRecord.add(str); + break; + case NUMERIC: + if (DateUtil.isCellDateFormatted(cell)) { + rowRecord.add(cell.getDateCellValue()); + break; + } else { + Double d = cell.getNumericCellValue(); + DecimalFormat numberFormat = new DecimalFormat("#.00"); + rowRecord.add(Double.parseDouble(numberFormat.format(d))); + } + break; + case BOOLEAN: + rowRecord.add(cell.getBooleanCellValue()); + break; + case FORMULA: + if (cell.getCachedFormulaResultType().equals(CellType.NUMERIC)) { + Double d = cell.getNumericCellValue() * 100; + DecimalFormat numberFormat = new DecimalFormat("#.00"); + rowRecord.add(Double.parseDouble(numberFormat.format(d))); + break; + } else if (cell.getCachedFormulaResultType().equals(CellType.STRING)) { + rowRecord.add(cell.getNumericCellValue()); + break; + } + break; + case BLANK: + rowRecord.add("-"); + break; + default: + rowRecord.add("no type match"); + break; + } + return rowRecord; + } + + private static boolean isMergedRegion(Workbook workbook, Sheet sheet, int cellRow, int cellColumn) { + int retVal = 0; + int sheetMergerCount = sheet.getNumMergedRegions(); + for (int i = 0; i < sheetMergerCount; i++) { + CellRangeAddress region = sheet.getMergedRegion(i); + int firstRow = region.getFirstRow(), firstCol = region.getFirstColumn(), lastRow = region.getLastRow(), + lastCol = region.getLastColumn(); + if (cellRow >= firstRow && cellRow <= lastRow) { + if (cellColumn >= firstCol && cellColumn <= lastCol) { + retVal = lastCol - firstCol + 1; + if (retVal > 0) { + return true; + } + break; + } + } + } + return false; + } + + private static Sheet getSheetMergerCellRegion(Workbook workbook, Sheet sheet, int cellRow, int cellColumn) { + int retVal = 0; + int sheetMergerCount = sheet.getNumMergedRegions(); + for (int i = 0; i < sheetMergerCount; i++) { + CellRangeAddress region = sheet.getMergedRegion(i); + int firstRow = region.getFirstRow(); + int firstCol = region.getFirstColumn(); + int lastRow = region.getLastRow(); + int lastCol = region.getLastColumn(); + if (cellRow >= firstRow && cellRow <= lastRow) { + if (cellColumn >= firstCol && cellColumn <= lastCol) { + retVal = lastCol - firstCol + 1; + Row row = sheet.getRow(i); + if (retVal > 0) { + for (int j = firstRow; j <= lastRow; j++) { + for (int k = firstCol; k <= lastCol; k++) { + Cell cell = sheet.getRow(region.getFirstRow()).getCell(region.getFirstColumn()); + String stringValue1 = null; + Double doubleValue1 = -1.00; + Date dateValue1 = new Date(); + Boolean booleanValue1 = false; + RichTextString formulaValue1 = null; + switch (cell.getCellType()) { + case STRING: + stringValue1 = sheet.getRow(region.getFirstRow()).getCell(region.getFirstColumn()) + .getRichStringCellValue().getString(); + sheet.getRow(j).getCell(k).setCellValue(stringValue1); + break; + case NUMERIC: + if (DateUtil.isCellDateFormatted( + sheet.getRow(region.getFirstRow()).getCell(region.getFirstColumn()))) { + dateValue1 = sheet.getRow(region.getFirstRow()).getCell(region.getFirstColumn()) + .getDateCellValue(); + sheet.getRow(j).getCell(k).setCellValue(dateValue1); + } else { + doubleValue1 = sheet.getRow(region.getFirstRow()) + .getCell(region.getFirstColumn()).getNumericCellValue(); + sheet.getRow(j).getCell(k).setCellValue(doubleValue1); + } + break; + case BOOLEAN: + booleanValue1 = sheet.getRow(region.getFirstRow()).getCell(region.getFirstColumn()) + .getBooleanCellValue(); + sheet.getRow(j).getCell(k).setCellValue(booleanValue1); + break; + case FORMULA: + if (cell.getCachedFormulaResultType().equals(CellType.NUMERIC)) { + doubleValue1 = cell.getNumericCellValue() * 100; + sheet.getRow(j).getCell(k).setCellValue(doubleValue1); + break; + } else if (cell.getCachedFormulaResultType().equals(CellType.STRING)) { + formulaValue1 = cell.getRichStringCellValue(); + sheet.getRow(j).getCell(k).setCellValue(formulaValue1); + break; + } + break; + case BLANK: + break; + default: + } + } + } + } + } + } + } + return sheet; + } + + private static void writeJsonArrayToFile(JSONArray data, String fileName) throws IOException { + String currentWorkingFolder = System.getProperty("user.dir"), + filePathSeperator = System.getProperty("file.separator"), + filePath = currentWorkingFolder + filePathSeperator + fileName; + BufferedWriter bufferedWriter = new BufferedWriter(new FileWriter(new File(filePath))); + bufferedWriter.write(data.toString()); + bufferedWriter.flush(); + bufferedWriter.close(); + } +} diff --git a/dashboard-ingest/src/main/java/com/ingestpipeline/util/ResponseCode.java b/dashboard-ingest/src/main/java/com/ingestpipeline/util/ResponseCode.java new file mode 100644 index 000000000..fe20fbf7f --- /dev/null +++ b/dashboard-ingest/src/main/java/com/ingestpipeline/util/ResponseCode.java @@ -0,0 +1,85 @@ +package com.ingestpipeline.util; + +/** + * + * @author Abhishek + * + */ +public enum ResponseCode { + UnAuthorised(Constants.UNAUTHORIZED_ID, Constants.UNAUTHORIZED), Success( + Constants.SUCCESS_ID, Constants.SUCCESS),FAILURE( + Constants.FAILURE_ID, Constants.PROCESS_FAIL); + /** + * error code contains int value + */ + private int errorCode; + /** + * errorMessage contains proper error message. + */ + private String errorMessage; + + + + /** + * @param errorCode + * @param errorMessage + */ + private ResponseCode(int errorCode, String errorMessage) { + this.errorCode = errorCode; + this.errorMessage = errorMessage; + } + + /** + * + * @param errorCode + * @return + */ + public String getMessage(int errorCode) { + return ""; + } + + /** + * @return + */ + public int getErrorCode() { + return errorCode; + } + + /** + * @param errorCode + */ + public void setErrorCode(int errorCode) { + this.errorCode = errorCode; + } + + /** + * @return + */ + public String getErrorMessage() { + return errorMessage; + } + + /** + * @param errorMessage + */ + public void setErrorMessage(String errorMessage) { + this.errorMessage = errorMessage; + } + + /** + * This method will provide status message based on code + * + * @param code + * @return String + */ + public static String getResponseMessage(int code) { + String value = ""; + ResponseCode responseCodes[] = ResponseCode.values(); + for (ResponseCode actionState : responseCodes) { + if (actionState.getErrorCode() == code) { + value = actionState.getErrorMessage(); + } + } + return value; + } +} diff --git a/dashboard-ingest/src/main/java/com/ingestpipeline/util/ResponseGenerator.java b/dashboard-ingest/src/main/java/com/ingestpipeline/util/ResponseGenerator.java new file mode 100644 index 000000000..46b73508c --- /dev/null +++ b/dashboard-ingest/src/main/java/com/ingestpipeline/util/ResponseGenerator.java @@ -0,0 +1,79 @@ +package com.ingestpipeline.util; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; + +public class ResponseGenerator { + + + private static ObjectMapper objectMapper = new ObjectMapper(); + + public static String failureResponse() throws JsonProcessingException{ + ObjectNode response = objectMapper.createObjectNode(); + response.put(JsonKey.STATUS_CODE, ResponseCode.FAILURE.getErrorCode()); + response.put(JsonKey.STATUS_MESSAGE, + ResponseCode.FAILURE.getErrorMessage()); + response.put(JsonKey.ERROR_MESSAGE, + ResponseCode.FAILURE.getErrorMessage()); + return JSONUtil.getJsonString(objectMapper,response); + } + + + public static String failureResponse(String message) throws JsonProcessingException{ + ObjectNode actualResponse = objectMapper.createObjectNode(); + + ObjectNode response = objectMapper.createObjectNode(); + response.put(JsonKey.STATUS_CODE, ResponseCode.FAILURE.getErrorCode()); + response.put(JsonKey.STATUS_MESSAGE, + ResponseCode.FAILURE.getErrorMessage()); + response.put(JsonKey.ERROR_MESSAGE,message); + actualResponse.putPOJO(JsonKey.STATUS,response); + + return JSONUtil.getJsonString(objectMapper,actualResponse); + } + + public static String feedbackFailureResponse(String message) throws JsonProcessingException{ + ObjectNode actualResponse = objectMapper.createObjectNode(); + + ObjectNode response = objectMapper.createObjectNode(); + response.put(JsonKey.STATUS_CODE, ResponseCode.FAILURE.getErrorCode()); + response.put(JsonKey.STATUS_MESSAGE, + ResponseCode.FAILURE.getErrorMessage()); + response.put(JsonKey.ERROR_MESSAGE,message); + actualResponse.putPOJO(JsonKey.STATUS,response); + + return JSONUtil.getJsonString(objectMapper,actualResponse); + } + + public static String successResponse(Object obj) throws JsonProcessingException { + ObjectNode actualResponse = objectMapper.createObjectNode(); + + ObjectNode response = objectMapper.createObjectNode(); + response.put(JsonKey.STATUS_CODE, ResponseCode.Success.getErrorCode()); + response.put(JsonKey.STATUS_MESSAGE, ResponseCode.Success.getErrorMessage()); + response.put(JsonKey.ERROR_MESSAGE, ""); + actualResponse.putPOJO(JsonKey.STATUS,response); + if (obj != null) { + actualResponse.putPOJO(JsonKey.RESPONSE_DATA, obj); + } + + return JSONUtil.getJsonString(objectMapper,actualResponse); + } + + public static String feedbackSuccessResponse(Object obj) throws JsonProcessingException { + ObjectNode response = objectMapper.createObjectNode(); + response.put(JsonKey.STATUS_CODE, ResponseCode.Success.getErrorCode()); + response.put(JsonKey.STATUS_MESSAGE, ResponseCode.Success.getErrorMessage()); + response.put(JsonKey.ERROR_MESSAGE, ""); + return JSONUtil.getJsonString(objectMapper,response); + } + + public static String feedbackSuccessResponse(String message) throws JsonProcessingException { + ObjectNode response = objectMapper.createObjectNode(); + response.put(JsonKey.STATUS_CODE, ResponseCode.Success.getErrorCode()); + response.put(JsonKey.STATUS_MESSAGE, ResponseCode.Success.getErrorMessage()); + response.put(JsonKey.ERROR_MESSAGE, ""); + return JSONUtil.getJsonString(objectMapper, response); + } +} diff --git a/dashboard-ingest/src/main/java/com/ingestpipeline/util/ResponseMessages.java b/dashboard-ingest/src/main/java/com/ingestpipeline/util/ResponseMessages.java new file mode 100644 index 000000000..5f75d4415 --- /dev/null +++ b/dashboard-ingest/src/main/java/com/ingestpipeline/util/ResponseMessages.java @@ -0,0 +1,11 @@ +package com.ingestpipeline.util; + +public interface ResponseMessages { + + public interface ErrorMessages { + } + + public interface SuccessMessages { + } + +} diff --git a/dashboard-ingest/src/main/java/com/ingestpipeline/util/ResponseUtil.java b/dashboard-ingest/src/main/java/com/ingestpipeline/util/ResponseUtil.java new file mode 100644 index 000000000..697a1de95 --- /dev/null +++ b/dashboard-ingest/src/main/java/com/ingestpipeline/util/ResponseUtil.java @@ -0,0 +1,91 @@ +package com.ingestpipeline.util; + +import java.io.IOException; + +import javax.servlet.http.HttpServletResponse; +import javax.ws.rs.WebApplicationException; +import javax.ws.rs.core.Response; +import javax.ws.rs.core.Response.ResponseBuilder; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.node.ObjectNode; + +/** + * + * @author Abhishek + * + */ + +public class ResponseUtil { + + /** + * Method to throw bad request with error message + * + * @param errorDescription + */ + public static void sendBadRequest(String errorDescription) { + ResponseBuilder resp = Response.status(Response.Status.BAD_REQUEST); + resp.entity(errorDescription); + WebApplicationException ex = new WebApplicationException(resp.build()); + throw ex; + } + + /** + * Method to throw Unauthorized request with error message + * + * @param errorDescription + *//* + public static Response sendUnauthorized(String errorDescription) throws JsonProcessingException{ + ResponseBuilder resp = Response.status(Response.Status.UNAUTHORIZED); + resp.entity(ResponseGenerator.failureResponse(errorDescription)); + return resp.build(); + }*/ + + /** + * Method to throw Internal server error + * + * @param errorDescription + */ + public static Response sendServerError(String errorDescription) { + ResponseBuilder resp = Response.status(Response.Status.INTERNAL_SERVER_ERROR); + resp.entity(errorDescription); + return resp.build(); + } + + /** + * Method to throw Unauthorized request with error message + * + * @param errorDescription + */ + public static void unauthorizedResponse(HttpServletResponse response, String errorDescription) throws IOException { + response.setStatus(HttpServletResponse.SC_UNAUTHORIZED); + response.setContentType("application/json"); + response.getWriter().write(errorDescription); + } + + public static Response sendOK(ObjectNode obj) { + ResponseBuilder resp = Response.status(Response.Status.OK); + resp.entity(obj); + return resp.build(); + } + + public static Response sendOK(String obj) { + ResponseBuilder resp = Response.status(Response.Status.OK); + resp.entity(obj); + return resp.build(); + } + + /*public static void sendInternalError(String errorDescription) throws JsonProcessingException { + ResponseBuilder resp = Response.status(Response.Status.INTERNAL_SERVER_ERROR); + resp.entity(ResponseGenerator.failureResponse(errorDescription)); + WebApplicationException ex = new WebApplicationException(resp.build()); + throw ex; + } + + public static void sendInternalError() throws JsonProcessingException { + ResponseBuilder resp = Response.status(Response.Status.INTERNAL_SERVER_ERROR); + resp.entity(ResponseGenerator.failureResponse()); + WebApplicationException ex = new WebApplicationException(resp.build()); + throw ex; + }*/ +} diff --git a/dashboard-ingest/src/main/resources/application.properties b/dashboard-ingest/src/main/resources/application.properties new file mode 100644 index 000000000..e588048c8 --- /dev/null +++ b/dashboard-ingest/src/main/resources/application.properties @@ -0,0 +1,68 @@ +#--------------------------- PATH & PORT CONFIGURATIONS ---------------------------# + +# SET CONTEXT PATH +server.port=8080 +server.servlet.context-path=/dashboard-ingest + +# KAFKA SERVER CONFIGURATIONS +spring.kafka.bootstrap.servers=localhost:9092 + +spring.kafka.consumer.value-deserializer=com.ingestpipeline.consumer.HashMapDeserializer +spring.kafka.consumer.key-deserializer=org.apache.kafka.common.serialization.StringDeserializer +spring.kafka.consumer.group-id=pipeline-group +spring.kafka.producer.key-serializer=org.apache.kafka.common.serialization.StringSerializer +spring.kafka.producer.value-serializer=org.springframework.kafka.support.serializer.JsonSerializer + +# KAFKA CONSUMER CONFIGURATIONS +kafka.consumer.config.auto_commit=true +kafka.consumer.config.auto_commit_interval=100 +kafka.consumer.config.session_timeout=15000 +kafka.consumer.config.group_id=pipeline-group +kafka.consumer.config.auto_offset_reset=earliest + +# KAFKA PRODUCER CONFIGURATIONS +kafka.producer.config.retries_config=0 +kafka.producer.config.batch_size_config=100000 +kafka.producer.config.linger_ms_config=100 +kafka.producer.config.buffer_memory_config=66554432 + +#----------------------------- ELASTIC SEARCH CONFIGURATIONS ------------------------------# +services.esindexer.host=http://104.211.240.72:9200/ +services.esindexer.host.name=104.211.240.72 +services.esindexer.host.port=9200 +services.esindexer.username=deploy +services.esindexer.password=Deploy123 +kafka.topics.incoming.data=tarento-dss-collection +kafka.topics.incoming.collection=dss-collection-update + +kafka.topics.enriched.data=egov-dss-ingest-enriched +kafka.topics.enriched.key=enrData +kafka.topics.bypass.update.data=egov-dss-update-receive +kafka.topics.bypass.update.post=egov-dss-update-post +kafka.topics.bypass.update.key=updateTopicKey +es.target.index.name = dss-target_v1 +es.index.name=dss-payment_v2 +es.index.type=general +es.document.type=pipeline_data +es.host.schema=https +es.push.direct=true +es.bypass.push.direct = true +es.index.searchQuery.collection = {\"size\": \"10\",\"query\":{\"bool\":{\"must\":[{\"wildcard\": {\"Data.tenantId.keyword\":\"pb.*\"}}]}}} +es.index.searchQuery.billing = {\"size\": \"10\",\"query\":{\"bool\":{\"must\":[{\"wildcard\": {\"tenantid.keyword\":\"pb.*\"}}]}}} +es.index.searchQuery.payment = {\"size\": \"10\"} +egov.services.esindexer.host.search=/_search + +spring.data.elasticsearch.cluster.name=elasticsearch-v1 +spring.data.elasticsearch.cluster.nodes=elasticsearch-v1 + +service.retry.maxAttempts=3 +service.retry.backoff.delay=1000 + +#----------------------------- Pipeline Schema Config Locations ------------------------------# +transformation.config.location=config/transform_transaction_v1.json + +schema.transaction.validation=config/validator_transaction_v1.json +schema.transaction.transformation=config/transform_transaction_v1.json +schema.transaction.enrichment=config/enrichment_transaction_v1.json + +config.schema.paths=classpath*:config/*.json diff --git a/dashboard-ingest/src/main/resources/config/CollectionDomainConfig.json b/dashboard-ingest/src/main/resources/config/CollectionDomainConfig.json new file mode 100644 index 000000000..5265a5350 --- /dev/null +++ b/dashboard-ingest/src/main/resources/config/CollectionDomainConfig.json @@ -0,0 +1,51 @@ +{ + "domainConfig": [ + { + "id": "1", + "businessType": "PT", + "indexName": "ptindex-v1", + "documentType": "", + "query": "{\"query\":{\"bool\":{\"must\":[{\"match_phrase\":{\"Data.propertyDetails.assessmentNumber\":\"value\"}},{\"match_phrase\":{\"Data.propertyId\":\"value\"}}]}}}", + "references": [ + { + "fieldName": "consumerCode", + "argument": "Data.propertyId", + "dataType": "String" + }, + { + "fieldName": "consumerCode", + "argument": "Data.propertyDetails.assessmentNumber", + "dataType": "String" + } + ] + }, + { + "id": "2", + "businessType": "TL", + "indexName": "tlindex-v1", + "documentType": "", + "query": "{\"query\":{\"bool\":{\"must\":[{\"match_phrase\":{\"Data.tradelicense.applicationnumber\":\"value\"}}]}}}", + "references": [ + { + "fieldName": "consumerCode", + "argument": "Data.tradelicense.applicationnumber", + "dataType": "String" + } + ] + }, + { + "id": "3", + "businessType": "PGR", + "indexName": "pgrindex-v1", + "documentType": "", + "query": "{}", + "references": [ + { + "fieldName": "consumerCode", + "argument": "complainNumber", + "dataType": "String" + } + ] + } + ] +} \ No newline at end of file diff --git a/dashboard-ingest/src/main/resources/config/DigressionPoints.json b/dashboard-ingest/src/main/resources/config/DigressionPoints.json new file mode 100644 index 000000000..2ebf20de3 --- /dev/null +++ b/dashboard-ingest/src/main/resources/config/DigressionPoints.json @@ -0,0 +1,8 @@ +{ + "digressionPoints": [ + { + "dataContext": "collection", + "paths": [ "Bill", "billDetails" ] + } + ] +} \ No newline at end of file diff --git a/dashboard-ingest/src/main/resources/config/DomainConfig.json b/dashboard-ingest/src/main/resources/config/DomainConfig.json new file mode 100644 index 000000000..17079e97f --- /dev/null +++ b/dashboard-ingest/src/main/resources/config/DomainConfig.json @@ -0,0 +1,92 @@ +{ + "domainConfig": [ + { + "id": "1", + "domain": "PT", + "indexName": "ptindex-v1", + "documentType": "", + "query": "{\"query\":{\"bool\":{\"must\":[{\"match_phrase\":{\"Data.propertyDetails.assessmentNumber\":\"value\"}},{\"match_phrase\":{\"Data.propertyId\":\"value\"}}]}}}", + "targetReferences": [ + { + "fieldName": "propertyId", + "argument": "Data.propertyId", + "dataType": "String", + "value": "$value", + "seperator": " ", + "expression": "propertyId" + }, + { + "fieldName": "assessmentNumber", + "argument": "Data.propertyDetails.assessmentNumber", + "dataType": "String", + "value": "$value", + "seperator": " ", + "expression": "assessmentNumber" + } + ], + "sourceReferences": [ + { + "fieldName": "consumerCode", + "argument": "Bill.billDetails.consumerCode", + "dataType": "String", + "value": "$value", + "seperator": ":", + "expression": "propertyId:assessmentNumber" + } + ] + }, + { + "id": "2", + "domain": "TL", + "indexName": "tlindex-v1", + "documentType": "", + "query": "{\"query\":{\"bool\":{\"must\":[{\"match_phrase\":{\"Data.tradelicense.applicationnumber\":\"value\"}}]}}}", + "targetReferences": [ + { + "fieldName": "applicationnumber", + "argument": "Data.tradelicense.applicationnumber", + "dataType": "String", + "value": "$value", + "seperator": " ", + "expression": "applicationnumber" + } + ], + "sourceReferences": [ + { + "fieldName": "consumerCode", + "argument": "Data.tradelicense.applicationnumber", + "dataType": "String", + "value": "$value", + "seperator": " ", + "expression": "applicationnumber" + } + ] + }, + { + "id": "3", + "domain": "PGR", + "indexName": "pgrindex-v1", + "documentType": "", + "query": "{}", + "targetReferences": [ + { + "fieldName": "complainNumber", + "argument": "Data.complainNumber", + "dataType": "String", + "value": "$value", + "seperator": " ", + "expression": "complainNumber" + } + ], + "sourceReferences": [ + { + "fieldName": "consumerCode", + "argument": "Data.complainNumber", + "value": "$value", + "seperator": " ", + "expression": "complainNumber" + } + ] + } + ] +} \ No newline at end of file diff --git a/dashboard-ingest/src/main/resources/config/EnhanceDomainConfig.json b/dashboard-ingest/src/main/resources/config/EnhanceDomainConfig.json new file mode 100644 index 000000000..d157119fb --- /dev/null +++ b/dashboard-ingest/src/main/resources/config/EnhanceDomainConfig.json @@ -0,0 +1,31 @@ +{ + "domainConfig": [ + { + "id": "1", + "domain": "PT", + "indexName": "property-assessments", + "documentType": "", + "query": "{\"query\":{\"bool\":{\"must\":[{\"match_phrase\":{\"Data.propertyId\":\"value\"}}]}}}", + "targetReferences": [ + { + "fieldName": "propertyId", + "argument": "Data.propertyId", + "dataType": "String", + "value": "$value", + "seperator": " ", + "expression": "propertyId" + } + ], + "sourceReferences": [ + { + "fieldName": "consumerCode", + "argument": "paymentDetails.bill.consumerCode", + "dataType": "String", + "value": "$value", + "seperator": ":", + "expression": "propertyId" + } + ] + } + ] +} diff --git a/dashboard-ingest/src/main/resources/config/TopicContextConfiguration.json b/dashboard-ingest/src/main/resources/config/TopicContextConfiguration.json new file mode 100644 index 000000000..beb133611 --- /dev/null +++ b/dashboard-ingest/src/main/resources/config/TopicContextConfiguration.json @@ -0,0 +1,14 @@ +{ + "topicContextConfigurations": [ + { + "topic": "dss-collection-update", + "dataContext": "collection", + "dataContextVersion": "v1" + }, + { + "topic": "topicTwo", + "dataContext": "billing", + "dataContextVersion": "v1" + } + ] +} \ No newline at end of file diff --git a/dashboard-ingest/src/main/resources/config/config.properties b/dashboard-ingest/src/main/resources/config/config.properties new file mode 100644 index 000000000..df830f5b1 --- /dev/null +++ b/dashboard-ingest/src/main/resources/config/config.properties @@ -0,0 +1,18 @@ +#--------------------------- Kafka Topics & Configurations ---------------------------# +dataContext=transaction + +kafka.transaction.ingest.topic=ingestData +kafka.transaction.ingest.key=ingestKey + +kafka.transaction.validation.topic=validData +kafka.transaction.validation.key=validKey + +kafka.transaction.transformation.topic=transformedData +kafka.transaction.transformation.key=transformedKey + +kafka.transaction.enrichment.topic=egov-dss-ingest-enriched +kafka.transaction.enrichment.key=enrichedKey + +pipelinerules={'VALIDATE':false,'TRANSFORM':true,'ENRICH':true} + +transformation.config.location=config/transform_transaction_v1.json diff --git a/dashboard-ingest/src/main/resources/config/enrichment_transaction_v1.json b/dashboard-ingest/src/main/resources/config/enrichment_transaction_v1.json new file mode 100644 index 000000000..038c1be33 --- /dev/null +++ b/dashboard-ingest/src/main/resources/config/enrichment_transaction_v1.json @@ -0,0 +1,49 @@ +{ + "dataCollectorName": "UserInformation", + "collectionSource": "UserService", + "basePath": "http://rain.idc.tarento.com/user/", + "collectionPoints": [ + { + "name": "UserMetaInfo", + "extensionPath": "/getUserMetaInfo", + "paramsPassed": [ + { + "sourceParam": "Transaction.KioskOperator.Id", + "pathLabel": "id", + "dataType": "Long", + "defaultValue": "0" + }, + { + "sourceParam": null, + "pathLabel": "isActive", + "dataType": "Boolean", + "defaultValue": "TRUE" + } + ], + "responseStructure": "{}", + "responseMapping": [ + { + + } + ] + }, + { + "name": "UserRoleInfo", + "extensionPath": "/getUserRole", + "paramsPassed": [ + { + "sourceParam": "Transaction.KioskOperator.Id", + "pathLabel": "userId", + "dataType": "Long", + "defaultValue": "0" + }, + { + "sourceParam": null, + "pathLabel": "isActive", + "dataType": "Boolean", + "defaultValue": "TRUE" + } + ] + } + ] +} diff --git a/dashboard-ingest/src/main/resources/config/transform_collection_v1.json b/dashboard-ingest/src/main/resources/config/transform_collection_v1.json new file mode 100644 index 000000000..3bed7dd59 --- /dev/null +++ b/dashboard-ingest/src/main/resources/config/transform_collection_v1.json @@ -0,0 +1,98 @@ +[ + { + "operation": "shift", + "spec": { + "tenantId": "tenantId", + "transactionId": "transactionId", + "@timestamp": "@timestamp", + "Bill": { + "$i": { + "billDetails": { + "$j": { + "voucherHeader": "Bill.billDetails.voucherHeader", + "boundary": "Bill.billDetails.boundary", + "collectedAmount": "Bill.billDetails.collectedAmount", + "channel": "Bill.billDetails.channel", + "manualReceiptDate": "Bill.billDetails.manualReceiptDate", + "expiryDate": "Bill.billDetails.expiryDate", + "demandId": "Bill.billDetails.demandId", + "amountPaid": "Bill.billDetails.amountPaid", + "consumerCode": "Bill.billDetails.consumerCode", + "id": "Bill.billDetails.id", + "department": "Bill.billDetails.department", + "receiptNumber": "Bill.billDetails.receiptNumber", + "businessService": "Bill.billDetails.businessService", + "receiptType": "Bill.billDetails.receiptType", + "fromPeriod": "Bill.billDetails.fromPeriod", + "stateId": "Bill.billDetails.stateId", + "receiptDate": "Bill.billDetails.receiptDate", + "bill": "Bill.billDetails.bill", + "billDate": "Bill.billDetails.billDate", + "consumerType": "Bill.billDetails.consumerType", + "totalAmount": "Bill.billDetails.totalAmount", + "collectionType": "Bill.billDetails.collectionType", + "fund": "Bill.billDetails.fund", + "manualReceiptNumber": "Bill.billDetails.manualReceiptNumber", + "tenantId": "Bill.billDetails.tenantId", + "toPeriod": "Bill.billDetails.toPeriod", + "partPaymentAllowed": "Bill.billDetails.partPaymentAllowed", + "billNumber": "Bill.billDetails.billNumber", + "status": "Bill.billDetails.status", + "billAccountDetails": { + "*": { + "taxHeadCode": "Bill.billDetails.billAccountDetails.[&1].taxHeadCode", + "glcode": "Bill.billDetails.billAccountDetails.[&1].glcode", + "amount": "Bill.billDetails.billAccountDetails.[&1].amount", + "billDetail": "Bill.billDetails.billAccountDetails.[&1].billDetail", + "purpose": "Bill.billDetails.billAccountDetails.[&1].purpose", + "adjustedAmount": "Bill.billDetails.billAccountDetails.[&1].adjustedAmount", + "tenantId": "Bill.billDetails.billAccountDetails.[&1].tenantId", + "id": "Bill.billDetails.billAccountDetails.[&1].id", + "demandDetailId": "Bill.billDetails.billAccountDetails.[&1].demandDetailId" + } + } + } + }, + "isCancelled": "Bill.isCancelled", + "paidBy": "Bill.paidBy", + "mobileNumber": "Bill.mobileNumber", + "payerId": "Bill.payerId", + "isActive": "Bill.isActive", + "additionalDetails": "Bill.additionalDetails", + "payerAddress": "Bill.payerAddress", + "taxAndPayments": "Bill.taxAndPayments", + "auditDetails": "Bill.auditDetails", + "tenantId": "Bill.tenantId", + "payerName": "Bill.payerName", + "id": "Bill.id", + "payerEmail": "Bill.payerEmail" + } + }, + "instrument": { + "bankAccount": "instrument.bankAccount", + "transactionDateInput": "instrument.transactionDateInput", + "amount": "instrument.amount", + "instrumentType": { + "instrumentTypeProperties": "instrument.typeProperties", + "name": "instrument.typeName", + "id": "instrument.typeId" + }, + "instrumentNumber": "instrument.number" + }, + "tenantData": { + "code": "tenantData.code", + "name": "tenantData.name", + "type": "tenantData.type", + "city": { + "name": "tenantData.cityName", + "localName": "tenantData.cityLocalName", + "districtCode": "tenantData.cityDistrictCode", + "districtName": "tenantData.cityDistrictName", + "regionName": "tenantData.cityRegionName", + "ulbGrade": "tenantData.cityUlbGrade", + "code": "tenantData.cityCode" + } + } + } + } +] diff --git a/dashboard-ingest/src/main/resources/config/transform_collection_v2.json b/dashboard-ingest/src/main/resources/config/transform_collection_v2.json new file mode 100644 index 000000000..cfd516bf9 --- /dev/null +++ b/dashboard-ingest/src/main/resources/config/transform_collection_v2.json @@ -0,0 +1,100 @@ +[ + { + "operation": "shift", + "spec": { + "tenantId": "tenantId", + "id": "id", + "paymentMode": "paymentMode", + "paymentStatus": "paymentStatus", + "transactionId": "transactionNumber", + "paidBy": "paidBy", + "payer": "payer", + "paymentDetails": { + "$i": { + "businessService": "paymentDetails.businessService", + "receiptType": "paymentDetails.receiptType", + "totalDue": "paymentDetails.totalDue", + "receiptDate": "paymentDetails.receiptDate", + "bill": { + "billDetails": { + "$j": { + "boundary": "paymentDetails.bill.billDetails.boundary", + "amount": "paymentDetails.bill.billDetails.amount", + "fromPeriod": "paymentDetails.bill.billDetails.fromPeriod", + "voucherHeader": "paymentDetails.bill.billDetails.voucherHeader", + "channel": "paymentDetails.bill.billDetails.channel", + "additionalDetails": "paymentDetails.bill.billDetails.additionalDetails", + "cancellationRemarks": "paymentDetails.bill.billDetails.cancellationRemarks", + "manualReceiptDate": "paymentDetails.bill.billDetails.manualReceiptDate", + "expiryDate": "paymentDetails.bill.billDetails.expiryDate", + "collectionType": "paymentDetails.bill.billDetails.collectionType", + "displayMessage": "paymentDetails.bill.billDetails.displayMessage", + "demandId": "paymentDetails.bill.billDetails.demandId", + "amountPaid": "paymentDetails.bill.billDetails.amountPaid", + "manualReceiptNumber": "paymentDetails.bill.billDetails.manualReceiptNumber", + "billId": "paymentDetails.bill.billDetails.billId", + "auditDetails": "paymentDetails.bill.billDetails.auditDetails", + "tenantId": "paymentDetails.bill.billDetails.tenantId", + "toPeriod": "paymentDetails.bill.billDetails.toPeriod", + "billDescription": "paymentDetails.bill.billDetails.billDescription", + "id": "paymentDetails.bill.billDetails.id", + "callBackForApportioning": "paymentDetails.bill.billDetails.callBackForApportioning", + "billAccountDetails": { + "*": { + "amount": "paymentDetails.bill.billDetails.billAccountDetails.[&1].amount", + "taxHeadCode": "paymentDetails.bill.billDetails.billAccountDetails.[&1].taxHeadCode", + "purpose": "paymentDetails.bill.billDetails.billAccountDetails.[&1].purpose", + "adjustedAmount": "paymentDetails.bill.billDetails.billAccountDetails.[&1].adjustedAmount", + "auditDetails": "paymentDetails.bill.billDetails.billAccountDetails.[&1].auditDetails", + "tenantId": "paymentDetails.bill.billDetails.billAccountDetails.[&1].tenantId", + "id": "paymentDetails.bill.billDetails.billAccountDetails.[&1].id", + "billDetailId": "paymentDetails.bill.billDetails.billAccountDetails.[&1].billDetailId", + "additionalDetails": "paymentDetails.bill.billDetails.billAccountDetails.[&1].additionalDetails", + "demandDetailId": "paymentDetails.bill.billDetails.billAccountDetails.[&1].demandDetailId", + "order": "paymentDetails.bill.billDetails.billAccountDetails.[&1].order", + "isActualDemand": "paymentDetails.bill.billDetails.billAccountDetails.[&1].isActualDemand" + } + } + } + }, + "isAdvanceAllowed": "paymentDetails.bill.isAdvanceAllowed", + "businessService": "paymentDetails.bill.businessService", + "isCancelled": "paymentDetails.bill.isCancelled", + "collectionModesNotAllowed": "paymentDetails.bill.collectionModesNotAllowed", + "paidBy": "paymentDetails.bill.paidBy", + "mobileNumber": "paymentDetails.bill.mobileNumber", + "payerId": "paymentDetails.bill.payerId", + "reasonForCancellation": "paymentDetails.bill.reasonForCancellation", + "minimumAmountToBePaid": "paymentDetails.bill.minimumAmountToBePaid", + "billDate": "paymentDetails.bill.billDate", + "additionalDetails": "paymentDetails.bill.additionalDetails", + "totalAmount": "paymentDetails.bill.totalAmount", + "payerAddress": "paymentDetails.bill.payerAddress", + "auditDetails": "paymentDetails.bill.auditDetails", + "tenantId": "paymentDetails.bill.tenantId", + "payerName": "paymentDetails.bill.payerName", + "partPaymentAllowed": "paymentDetails.bill.partPaymentAllowed", + "consumerCode": "paymentDetails.bill.consumerCode", + "id": "paymentDetails.bill.id", + "billNumber": "paymentDetails.bill.billNumber", + "payerEmail": "paymentDetails.bill.payerEmail", + "status": "paymentDetails.bill.status" + }, + "totalAmountPaid": "paymentDetails.totalAmountPaid", + "additionalDetails": "paymentDetails.additionalDetails", + "manualReceiptDate": "paymentDetails.manualReceiptDate", + "manualReceiptNumber": "paymentDetails.manualReceiptNumber", + "billId": "paymentDetails.billId", + "auditDetails": "paymentDetails.auditDetails", + "tenantId": "paymentDetails.tenantId", + "id": "paymentDetails.id", + "receiptNumber": "paymentDetails.receiptNumber" + } + }, + "auditDetails": "auditDetails", + "additionalDetails": "additionalDetails", + "tenantData": "tenantData", + "@timestamp": "@timestamp" + } + } +] diff --git a/dashboard-ingest/src/main/resources/config/transform_pgr_v1.json b/dashboard-ingest/src/main/resources/config/transform_pgr_v1.json new file mode 100644 index 000000000..e1d93fe4c --- /dev/null +++ b/dashboard-ingest/src/main/resources/config/transform_pgr_v1.json @@ -0,0 +1,12 @@ +[ + { + "operation": "shift", + "spec": { + "tenantId": "tenantId", + "serviceRequestId": "serviceRequestId", + "complainCategory": "complainCategory", + "dateOfComplaint": "dateOfComplaint" + + } + } +] diff --git a/dashboard-ingest/src/main/resources/config/transform_pt_enhance_v1.json b/dashboard-ingest/src/main/resources/config/transform_pt_enhance_v1.json new file mode 100644 index 000000000..722d01a3a --- /dev/null +++ b/dashboard-ingest/src/main/resources/config/transform_pt_enhance_v1.json @@ -0,0 +1,12 @@ +[ + { + "operation": "shift", + "spec": { + "assessmentNumber": "assessmentNumber", + "assessmentDate": "assessmentDate", + "financialYear": "financialYear", + "status" : "status", + "propertyId": "propertyId" + } + } +] diff --git a/dashboard-ingest/src/main/resources/config/transform_pt_v1.json b/dashboard-ingest/src/main/resources/config/transform_pt_v1.json new file mode 100644 index 000000000..1bc6766b7 --- /dev/null +++ b/dashboard-ingest/src/main/resources/config/transform_pt_v1.json @@ -0,0 +1,11 @@ +[ + { + "operation": "shift", + "spec": { + "tenantId": "tenantId", + "propertyId": "propertyId", + "propertyDetails": "propertyDetails", + "ward" : "ward" + } + } +] diff --git a/dashboard-ingest/src/main/resources/config/transform_target_v1.json b/dashboard-ingest/src/main/resources/config/transform_target_v1.json new file mode 100644 index 000000000..da01e1ffe --- /dev/null +++ b/dashboard-ingest/src/main/resources/config/transform_target_v1.json @@ -0,0 +1,20 @@ +[ + { + "operation": "shift", + "spec": { + "*": { + "S. No.": "[&1].snoForMunicipalCorporation", + "Tenant Id": "[&1].tenantIdForMunicipalCorporation", + "ULB Name": "[&1].ulbName", + "Actual Collection (in Lakhs)": "[&1].actualCollectionForMunicipalCorporation", + "Budget Proposed (in Lakhs)": "[&1].budgetProposedForMunicipalCorporation", + "% Actual Collection over Budgeted": "[&1].actualCollectionBudgetedForMunicipalCorporation", + "Financial Year": "[&1].financialYear", + "Timestamp": "[&1].Timestamp", + "Sheet Name": "[&1].businessService" + } + } + } +] + + diff --git a/dashboard-ingest/src/main/resources/config/transform_tl_v1.json b/dashboard-ingest/src/main/resources/config/transform_tl_v1.json new file mode 100644 index 000000000..4f6ae47e9 --- /dev/null +++ b/dashboard-ingest/src/main/resources/config/transform_tl_v1.json @@ -0,0 +1,28 @@ +[ + { + "operation": "shift", + "spec": { + "tradelicense": { + "tenantid": "tenantId", + "calculation": "tradeLicense.calculation", + "licensenumber": "tradeLicense.licenseNumber", + "commencementdate": "tradeLicense.commencementDate", + "licensetype": "tradeLicense.licenseType", + "financialyear": "tradeLicense.finYear", + "tradename": "tradeLicense.tradeName", + "applicationnumber": "tradeLicense.applicationNumber", + "accountid": "tradeLicense.accountId", + "applicationdate": "tradeLicense.applicationDate", + "oldpropertyid": "tradeLicense.oldPropertyId", + "action": "tradeLicense.action" + }, + "ward": "ward" + } + } +] + + + + + + diff --git a/dashboard-ingest/src/main/resources/config/transform_transaction_v1.json b/dashboard-ingest/src/main/resources/config/transform_transaction_v1.json new file mode 100644 index 000000000..0fd6b03ef --- /dev/null +++ b/dashboard-ingest/src/main/resources/config/transform_transaction_v1.json @@ -0,0 +1,29 @@ + [ + { + "operation": "shift", + "spec": { + "rating": { + "primary": { + "value": "Rating", + "max": "RatingRange" + }, + "*": { + "max": "SecondaryRatings.&1.Range", + "value": "SecondaryRatings.&1.Value", + "$": "SecondaryRatings.&1.Id" + } + } + } + }, + { + "operation": "default", + "spec": { + "Range": 5, + "SecondaryRatings": { + "*": { + "Range": 5 + } + } + } + } + ] diff --git a/dashboard-ingest/src/main/resources/config/validator_transaction_v1.json b/dashboard-ingest/src/main/resources/config/validator_transaction_v1.json new file mode 100644 index 000000000..e451ebda5 --- /dev/null +++ b/dashboard-ingest/src/main/resources/config/validator_transaction_v1.json @@ -0,0 +1,32 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "title": "transaction", + "description": "Each Transaction has to go through this validation conditions", + "type": "object", + "items": { + "type": "object", + "properties": { + "rating": { + "type": "object", + "items": { + "type": "object", + "properties": { + "primary": { + "type": "object", + "properties": { + "value": { + "type": "integer", + "minimum": 1, + "maximum": 5 + } + } + } + } + } + } + } + }, + "required": [ + "rating" + ] +} \ No newline at end of file diff --git a/dashboard-ingest/src/main/resources/data.sql b/dashboard-ingest/src/main/resources/data.sql new file mode 100644 index 000000000..e8d22e21f --- /dev/null +++ b/dashboard-ingest/src/main/resources/data.sql @@ -0,0 +1,13 @@ +/*DROP TABLE IF EXISTS targetcollection; + +CREATE TABLE targetcollection ( + Serial Number INT PRIMARY KEY, + City Name VARCHAR(250) NOT NULL, + Code VARCHAR(250) NOT NULL, + Budget Proposed(in Lakhs) VARCHAR(250) NOT NULL, + Actual Collection(in Lakhs) VARCHAR(250) NOT NULL + % Actual Collection over Budgeted VARCHAR(250) NOT NULL +); + +INSERT INTO targetcollection (Serial Number, City Name, Code, Budget Proposed(in Lakhs), Actual Collection(in Lakhs),% Actual Collection over Budgeted) VALUES + (?,?,?,?,?,?)*/ \ No newline at end of file diff --git a/dashboard-ingest/src/test/java/com/ingestpipeline/testcases/IngestTestClient.java b/dashboard-ingest/src/test/java/com/ingestpipeline/testcases/IngestTestClient.java new file mode 100644 index 000000000..ea23eb13b --- /dev/null +++ b/dashboard-ingest/src/test/java/com/ingestpipeline/testcases/IngestTestClient.java @@ -0,0 +1,54 @@ +package com.ingestpipeline.testcases; + +import java.util.LinkedHashMap; +import java.util.List; + +import org.springframework.web.client.RestTemplate; + + +public class IngestTestClient { + + public static final String REST_SERVICE_URI = "http://localhost:8081/ingest/api"; + + /* GET */ + @SuppressWarnings("unchecked") + private static void listAllUsers(){ + System.out.println("Testing listAllUsers API-----------"); + + RestTemplate restTemplate = new RestTemplate(); + List> usersMap = restTemplate.getForObject(REST_SERVICE_URI+"/user/", List.class); + + if(usersMap!=null){ + for(LinkedHashMap map : usersMap){ + System.out.println("User : id="+map.get("id")+", Name="+map.get("name")+", Age="+map.get("age")+", Salary="+map.get("salary"));; + } + }else{ + System.out.println("No user exist----------"); + } + } + + /* DELETE */ + private static void deleteUser() { + System.out.println("Testing delete User API----------"); + RestTemplate restTemplate = new RestTemplate(); + restTemplate.delete(REST_SERVICE_URI+"/user/3"); + } + + + /* DELETE */ + private static void deleteAllUsers() { + System.out.println("Testing all delete Users API----------"); + RestTemplate restTemplate = new RestTemplate(); + restTemplate.delete(REST_SERVICE_URI+"/user/"); + } + + public static void main(String args[]){ + listAllUsers(); + listAllUsers(); + listAllUsers(); + //deleteUser(); + listAllUsers(); + //deleteAllUsers(); + listAllUsers(); + } +} \ No newline at end of file