Skip to content

Commit 4827262

Browse files
pbartuschmkr
pbartusch
and
mkr
authored
Rules usage report (#146)
* Update to Angular 13 and Node 16 * Adding a report to show the usage of rules as imported from web analytics --------- Co-authored-by: mkr <code@mkr.io>
1 parent c341912 commit 4827262

25 files changed

+452
-50
lines changed

‎.gitignore

+3
Original file line numberDiff line numberDiff line change
@@ -31,3 +31,6 @@ hs_err_pid*
3131

3232
# SBT related stuff
3333
.bsp
34+
35+
# Ignore the Angular Cache
36+
frontend/.angular/cache

‎Dockerfile

+3-2
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
# syntax = docker/dockerfile:1.0-experimental
22
FROM openjdk:11-buster as builder
33

4-
ARG NODE_VERSION=10
4+
ARG NODE_VERSION=16
55

66
RUN echo "deb https://repo.scala-sbt.org/scalasbt/debian /" | tee -a /etc/apt/sources.list.d/sbt.list \
77
&& curl -sL "https://keyserver.ubuntu.com/pks/lookup?op=get&search=0x2EE0EA64E40A89B84B2DF73499E82A75642AC823" | apt-key add \
@@ -14,7 +14,8 @@ RUN apt-get install -y lsb-release \
1414
&& echo "deb-src https://deb.nodesource.com/node_$NODE_VERSION.x $DISTRO main" >> /etc/apt/sources.list.d/nodesource.list \
1515
&& curl -sSL https://deb.nodesource.com/gpgkey/nodesource.gpg.key | apt-key add - \
1616
&& apt-get update \
17-
&& apt-get install -y nodejs
17+
&& apt-get install -y nodejs \
18+
&& apt-get install -y g++ make
1819

1920
COPY . /smui
2021
WORKDIR /smui

‎app/controllers/ApiController.scala

+14-2
Original file line numberDiff line numberDiff line change
@@ -21,13 +21,14 @@ import models.config.SmuiVersion
2121
import models.config.TargetEnvironment._
2222
import models.input.{InputTagId, InputValidator, ListItem, SearchInputId, SearchInputWithRules}
2323
import models.querqy.QuerqyRulesTxtGenerator
24+
import models.reports.RulesUsageReport
2425
import models.rules.{DeleteRule, FilterRule, RedirectRule, SynonymRule, UpDownRule}
2526
import models.spellings.{CanonicalSpellingId, CanonicalSpellingValidator, CanonicalSpellingWithAlternatives}
2627
import org.pac4j.core.profile.{ProfileManager, UserProfile}
2728
import org.pac4j.play.PlayWebContext
2829
import org.pac4j.play.scala.{Security, SecurityComponents}
2930
import play.api.libs.Files
30-
import services.{RulesTxtDeploymentService, RulesTxtImportService}
31+
import services.{RulesTxtDeploymentService, RulesTxtImportService, RulesUsageService}
3132

3233

3334
// TODO Make ApiController pure REST- / JSON-Controller to ensure all implicit Framework responses (e.g. 400, 500) conformity
@@ -37,7 +38,8 @@ class ApiController @Inject()(val controllerComponents: SecurityComponents,
3738
querqyRulesTxtGenerator: QuerqyRulesTxtGenerator,
3839
rulesTxtDeploymentService: RulesTxtDeploymentService,
3940
rulesTxtImportService: RulesTxtImportService,
40-
targetEnvironmentConfigService: TargetEnvironmentConfigService)
41+
targetEnvironmentConfigService: TargetEnvironmentConfigService,
42+
rulesUsageService: RulesUsageService)
4143
(implicit executionContext: ExecutionContext)
4244
extends Security[UserProfile] with play.api.i18n.I18nSupport with Logging {
4345

@@ -667,6 +669,16 @@ class ApiController @Inject()(val controllerComponents: SecurityComponents,
667669
}
668670
}
669671

672+
def getRulesUsageReport(solrIndexId: String): Action[AnyContent] = Action {
673+
rulesUsageService.getRulesUsageStatistics.map { ruleUsageStatistics =>
674+
val allSearchInputs = searchManagementRepository.listAllSearchInputsInclDirectedSynonyms(SolrIndexId(solrIndexId))
675+
val report = RulesUsageReport.create(allSearchInputs, ruleUsageStatistics)
676+
Ok(Json.toJson(report))
677+
}.getOrElse(
678+
NoContent
679+
)
680+
}
681+
670682
private def lookupUserInfo(request: Request[AnyContent]) = {
671683
val maybeUserId = getProfiles(request).headOption.map(_.getId)
672684
logger.debug(s"Current user: $maybeUserId")

‎app/models/FeatureToggleModel.scala

+8-1
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ import play.twirl.api.utils.StringEscapeUtils
77

88
import scala.util.Try
99
import models.rules.UpDownRule
10+
import services.RulesUsageService
1011

1112
// TODO refactor FeatureToggleModel (and FeatureToggleService) to config package (for being in sync with Spec structure)
1213
package object FeatureToggleModel extends Logging {
@@ -65,6 +66,7 @@ package object FeatureToggleModel extends Logging {
6566
private val FEATURE_CUSTOM_UP_DOWN_MAPPINGS = "toggle.ui-concept.custom.up-down-dropdown-mappings"
6667
private val SMUI_DEPLOYMENT_GIT_REPO_URL = "smui.deployment.git.repo-url"
6768
private val SMUI_DEPLOYMENT_GIT_FN_COMMON_RULES_TXT = "smui2solr.deployment.git.filename.common-rules-txt"
69+
private val FEATURE_TOGGLE_REPORT_RULE_USAGE_STATISTICS = "toggle.report.rule-usage-statistics"
6870

6971
/**
7072
* helper for custom UP/DOWN mappings
@@ -166,7 +168,8 @@ package object FeatureToggleModel extends Logging {
166168
JsFeatureToggle(FEATURE_TOGGLE_DEPLOYMENT_LABEL, new JsStringFeatureToggleValue(
167169
appConfig.getOptional[String](FEATURE_TOGGLE_DEPLOYMENT_LABEL).getOrElse("LIVE"))),
168170
JsFeatureToggle(FEATURE_TOGGLE_DEPLOYMENT_PRELIVE_LABEL, new JsStringFeatureToggleValue(
169-
appConfig.getOptional[String](FEATURE_TOGGLE_DEPLOYMENT_PRELIVE_LABEL).getOrElse("PRELIVE")))
171+
appConfig.getOptional[String](FEATURE_TOGGLE_DEPLOYMENT_PRELIVE_LABEL).getOrElse("PRELIVE"))),
172+
jsBoolFeatureToggle(FEATURE_TOGGLE_REPORT_RULE_USAGE_STATISTICS, hasRulesUsageStatisticsLocationConfigured)
170173
)
171174
}
172175

@@ -230,6 +233,10 @@ package object FeatureToggleModel extends Logging {
230233
appConfig.getOptional[String](FEATURE_TOGGLE_DEPLOYMENT_PRELIVE_LABEL).getOrElse("PRELIVE")
231234
}
232235

236+
def hasRulesUsageStatisticsLocationConfigured: Boolean = {
237+
appConfig.getOptional[String](RulesUsageService.ConfigKeyRuleUsageStatistics).nonEmpty
238+
}
239+
233240
}
234241

235242
}

‎app/models/reports/RulesUsageReport.scala

+56
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
package models.reports
2+
3+
import models.input.{SearchInput, SearchInputId, SearchInputWithRules}
4+
import play.api.Logging
5+
import play.api.libs.json.{Json, OFormat}
6+
import services.RulesUsage
7+
8+
case class RulesUsageReportEntry(
9+
// the search input ID from the report
10+
searchInputId: String,
11+
// the stored search input term if it could be found in the DB
12+
searchInputTerm: Option[String],
13+
// the user keywords/query that the search input was triggered on
14+
keywords: String,
15+
// the frequency that rule was triggered for the particular keywords
16+
frequency: Int
17+
)
18+
19+
case class RulesUsageReport(items: Seq[RulesUsageReportEntry])
20+
21+
object RulesUsageReport extends Logging {
22+
23+
implicit val jsonFormatRulesUsageReportEntry: OFormat[RulesUsageReportEntry] = Json.format[RulesUsageReportEntry]
24+
implicit val jsonFormatRulesUsageReport: OFormat[RulesUsageReport] = Json.format[RulesUsageReport]
25+
26+
def create(searchInputs: Seq[SearchInputWithRules], rulesUsageStatistics: Seq[RulesUsage]): RulesUsageReport = {
27+
// perform a "full outer join" of the rules usage with the existing search inputs
28+
val searchInputsById = searchInputs.map(searchInput => searchInput.id.id -> searchInput).toMap
29+
val searchInputIdsFromAnalytics = rulesUsageStatistics.map(_.inputId.id).toSet
30+
val searchInputIdsNotFound = searchInputIdsFromAnalytics -- searchInputsById.keySet
31+
val searchInputIdsNotUsed = searchInputsById.keySet -- searchInputIdsFromAnalytics
32+
logger.info(s"Creating report from ${searchInputIdsFromAnalytics.size} used search inputs" +
33+
s" and ${searchInputsById.size} search inputs currently configured" +
34+
s" with ${searchInputIdsNotFound.size} search inputs not found" +
35+
s" and ${searchInputIdsNotUsed.size} search inputs not used")
36+
37+
val reportEntriesUsedSearchInputs = rulesUsageStatistics.map { rulesUsage: RulesUsage =>
38+
RulesUsageReportEntry(
39+
rulesUsage.inputId.id,
40+
searchInputsById.get(rulesUsage.inputId.id).map(_.term),
41+
rulesUsage.keywords,
42+
rulesUsage.frequency
43+
)
44+
}
45+
val reportEntriesUnusedSearchInputs = searchInputIdsNotUsed.map { searchInputId =>
46+
RulesUsageReportEntry(
47+
searchInputId,
48+
searchInputsById.get(searchInputId).map(_.term),
49+
"",
50+
0
51+
)
52+
}
53+
RulesUsageReport(reportEntriesUsedSearchInputs ++ reportEntriesUnusedSearchInputs)
54+
}
55+
56+
}

‎app/services/ReaderProvider.scala

+50
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
package services
2+
3+
import com.google.cloud.storage.{BlobId, StorageOptions}
4+
5+
import java.io.{InputStreamReader, Reader}
6+
import java.net.URI
7+
import java.nio.channels.Channels
8+
import java.nio.charset.StandardCharsets
9+
import java.nio.file.{Files, Paths}
10+
11+
trait ReaderProvider {
12+
def openReader(url: String): Reader
13+
}
14+
15+
class FileSystemReaderProvider extends ReaderProvider {
16+
override def openReader(url: String): Reader = {
17+
val uri = if (url.startsWith("file://")) url else "file://" + url
18+
val path = Paths.get(new URI(uri))
19+
Files.newBufferedReader(path)
20+
}
21+
}
22+
23+
class GcsReaderProvider extends ReaderProvider {
24+
25+
override def openReader(url: String): Reader = {
26+
val storage = StorageOptions.getDefaultInstance.getService
27+
val blob = storage.get(BlobId.fromGsUtilUri(url))
28+
val readChannel = blob.reader()
29+
new InputStreamReader(Channels.newInputStream(readChannel), StandardCharsets.UTF_8)
30+
}
31+
}
32+
33+
class ReaderProviderDispatcher extends ReaderProvider {
34+
35+
private val fileSystemReaderProvider = new FileSystemReaderProvider()
36+
private lazy val gcsReaderProvider: GcsReaderProvider = {
37+
new GcsReaderProvider()
38+
}
39+
40+
override def openReader(url: String): Reader = {
41+
val readerProvider = url.toLowerCase.trim match {
42+
case url if url.startsWith("gs://") => gcsReaderProvider
43+
case url if url.startsWith("file://") => fileSystemReaderProvider
44+
case url if Files.exists(Paths.get(url)) => fileSystemReaderProvider
45+
case _ => throw new IllegalArgumentException(s"Unsupported URL scheme or file not found: ${url}")
46+
}
47+
readerProvider.openReader(url)
48+
}
49+
50+
}

‎app/services/RulesUsageService.scala

+49
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
package services
2+
3+
import models.input.SearchInputId
4+
import org.apache.commons.csv.CSVFormat
5+
import play.api.{Configuration, Logging}
6+
7+
import javax.inject.Inject
8+
import scala.collection.JavaConverters.iterableAsScalaIterableConverter
9+
10+
case class RulesUsage(inputId: SearchInputId,
11+
keywords: String,
12+
frequency: Int)
13+
14+
class RulesUsageService @Inject()(configuration: Configuration,
15+
readerProvider: ReaderProviderDispatcher) extends Logging {
16+
17+
private val CsvFormat = CSVFormat.DEFAULT.builder().setHeader().setSkipHeaderRecord(true).build()
18+
19+
def getRulesUsageStatistics: Option[Seq[RulesUsage]] = {
20+
configuration.getOptional[String](RulesUsageService.ConfigKeyRuleUsageStatistics)
21+
.map { location =>
22+
logger.info(s"Loading rule usage statistics from ${location}")
23+
try {
24+
val reader = readerProvider.openReader(location)
25+
try {
26+
CsvFormat.parse(reader).asScala.map { record =>
27+
RulesUsage(
28+
SearchInputId(record.get("SMUI_GUID")),
29+
record.get("USER_QUERY"),
30+
record.get("FREQUENCY").toInt)
31+
}.toSeq
32+
} finally {
33+
reader.close()
34+
}
35+
} catch {
36+
case e: Exception =>
37+
logger.error("Could not load rule usage statistics", e)
38+
Seq.empty
39+
}
40+
}
41+
}
42+
43+
}
44+
45+
object RulesUsageService {
46+
47+
val ConfigKeyRuleUsageStatistics = "smui.rule-usage-statistics.location"
48+
49+
}

‎build.sbt

+3-1
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
import com.typesafe.sbt.GitBranchPrompt
22

33
name := "search-management-ui"
4-
version := "4.0.11"
4+
version := "4.1.0"
55
maintainer := "Contact productful.io <hello@productful.io>"
66

77
scalaVersion := "2.12.17"
@@ -67,10 +67,12 @@ libraryDependencies ++= {
6767
"com.typesafe.play" %% "play-json" % "2.9.3",
6868
"com.pauldijou" %% "jwt-play" % "4.1.0",
6969
"com.fasterxml.jackson.module" %% "jackson-module-scala" % JacksonVersion,
70+
"org.apache.commons" % "commons-csv" % "1.10.0",
7071
"org.apache.shiro" % "shiro-core" % "1.12.0",
7172
"org.pac4j" % "pac4j-http" % Pac4jVersion excludeAll (JacksonCoreExclusion, BcProv15Exclusion, SpringJclBridgeExclusion),
7273
"org.pac4j" % "pac4j-saml" % Pac4jVersion excludeAll (JacksonCoreExclusion, BcProv15Exclusion, SpringJclBridgeExclusion),
7374
"org.pac4j" %% "play-pac4j" % "11.1.0-PLAY2.8",
75+
"com.google.cloud" % "google-cloud-storage" % "2.33.0",
7476
"org.scalatestplus.play" %% "scalatestplus-play" % "3.1.0" % Test,
7577
"org.mockito" % "mockito-all" % "1.10.19" % Test,
7678
"com.pauldijou" %% "jwt-play" % "4.1.0",

‎conf/application.conf

+4
Original file line numberDiff line numberDiff line change
@@ -168,6 +168,10 @@ database.dispatcher {
168168
}
169169
}
170170

171+
smui.rule-usage-statistics {
172+
# Set a file:// or gs:// URL to a CSV file to load rule usage statistics for the rule usage report
173+
location = ${?SMUI_RULE_USAGE_STATISTICS_LOCATION}
174+
}
171175

172176
# Enable security module/filter
173177
play.modules.enabled += "modules.SecurityModule"

‎conf/routes

+1
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,7 @@ GET /api/v1/report/activity-report/:solrIndexId controllers.ApiC
4444
GET /api/v1/version/latest-info controllers.ApiController.getLatestVersionInfo()
4545
GET /api/v2/log/deployment-info controllers.ApiController.getLatestDeploymentResult(solrIndexId: String)
4646
GET /api/v1/config/target-environment controllers.ApiController.getTargetEnvironment()
47+
GET /api/v1/report/rules-usage-report/:solrIndexId controllers.ApiController.getRulesUsageReport(solrIndexId: String)
4748

4849
# Map static resources from the /public folder to the /assets URL path
4950
GET /*file controllers.FrontendController.assetOrDefault(file)

‎frontend/angular.json

+7-3
Original file line numberDiff line numberDiff line change
@@ -28,9 +28,13 @@
2828
],
2929
"styles": [
3030
"node_modules/bootstrap/dist/css/bootstrap.min.css",
31+
"node_modules/datatables.net-dt/css/dataTables.dataTables.css",
3132
"src/styles.css"
3233
],
33-
"scripts": []
34+
"scripts": [
35+
"node_modules/jquery/dist/jquery.js",
36+
"node_modules/datatables.net/js/dataTables.js"
37+
]
3438
},
3539
"configurations": {
3640
"production": {
@@ -50,8 +54,8 @@
5054
"budgets": [
5155
{
5256
"type": "initial",
53-
"maximumWarning": "500kb",
54-
"maximumError": "1mb"
57+
"maximumWarning": "1mb",
58+
"maximumError": "2mb"
5559
},
5660
{
5761
"type": "anyComponentStyle",

‎frontend/package.json

+26-20
Original file line numberDiff line numberDiff line change
@@ -13,35 +13,41 @@
1313
},
1414
"private": true,
1515
"dependencies": {
16-
"@angular/animations": "~11.0.1",
17-
"@angular/common": "~11.0.1",
18-
"@angular/compiler": "~11.0.1",
19-
"@angular/core": "~11.0.1",
20-
"@angular/forms": "~11.0.1",
21-
"@angular/localize": "~11.0.1",
22-
"@angular/platform-browser": "~11.0.1",
23-
"@angular/platform-browser-dynamic": "~11.0.1",
24-
"@angular/router": "~11.0.1",
16+
"@angular/animations": "~13.4.0",
17+
"@angular/common": "~13.4.0",
18+
"@angular/compiler": "~13.4.0",
19+
"@angular/core": "~13.4.0",
20+
"@angular/forms": "~13.4.0",
21+
"@angular/localize": "~13.4.0",
22+
"@angular/platform-browser": "~13.4.0",
23+
"@angular/platform-browser-dynamic": "~13.4.0",
24+
"@angular/router": "~13.4.0",
2525
"@fortawesome/fontawesome-free": "^5.15.1",
2626
"@ng-bootstrap/ng-bootstrap": "^8.0.0",
27-
"angular2-multiselect-dropdown": "^4.6.6",
27+
"angular-datatables": "^13.1.0",
28+
"angular2-multiselect-dropdown": "^5.0.4",
2829
"angular2-toaster": "^11.0.1",
2930
"bootstrap": "^4.5.0",
31+
"datatables.net": "~2.0.3",
32+
"datatables.net-dt": "~2.0.3",
33+
"jquery": "~3.7.1",
3034
"rxjs": "~6.6.0",
31-
"tslib": "^2.0.0",
35+
"tslib": "^2.6.2",
3236
"zone.js": "~0.10.2"
3337
},
3438
"devDependencies": {
35-
"@angular-devkit/build-angular": "~0.1100.2",
36-
"@angular-eslint/builder": "1.1.0",
37-
"@angular-eslint/eslint-plugin": "1.1.0",
38-
"@angular-eslint/eslint-plugin-template": "1.1.0",
39-
"@angular-eslint/schematics": "1.1.0",
40-
"@angular-eslint/template-parser": "1.1.0",
41-
"@angular/cli": "~11.0.2",
42-
"@angular/compiler-cli": "~11.0.1",
39+
"@angular-devkit/build-angular": "13.3.11",
40+
"@angular-eslint/builder": "13.4.0",
41+
"@angular-eslint/eslint-plugin": "13.4.0",
42+
"@angular-eslint/eslint-plugin-template": "13.4.0",
43+
"@angular-eslint/schematics": "13.4.0",
44+
"@angular-eslint/template-parser": "13.4.0",
45+
"@angular/cli": "13.3.11",
46+
"@angular/compiler-cli": "13.3.11",
47+
"@types/datatables.net": "1.10.21",
4348
"@types/jasmine": "~3.6.0",
4449
"@types/jasminewd2": "~2.0.8",
50+
"@types/jquery": "~3.5.29",
4551
"@types/node": "^12.11.1",
4652
"@typescript-eslint/eslint-plugin": "4.3.0",
4753
"@typescript-eslint/parser": "4.3.0",
@@ -59,6 +65,6 @@
5965
"karma-jasmine": "~4.0.0",
6066
"karma-jasmine-html-reporter": "^1.5.0",
6167
"ts-node": "~8.3.0",
62-
"typescript": "~4.0.2"
68+
"typescript": "~4.6.4"
6369
}
6470
}

0 commit comments

Comments
 (0)