TW-1442: Add API for sites blacklist for replacing ads (#156)

* TW-1442 Add API for sites blacklist for replacing ads

* TW-1442 Change response types for URLs blacklist

package.json

@@ -53,6 +53,7 @@
     "@types/express": "^4.17.17",
     "@types/express-jwt": "^7.4.2",
     "@types/express-unless": "^2.0.1",
+    "@types/lodash": "^4.17.0",
     "@types/memoizee": "^0.4.5",
     "@types/node": "^18.14.6",
     "@types/pino": "^6.3.8",

src/advertising/external-ads.ts

@@ -84,7 +84,7 @@ interface AdStylesOverrides {
   style: Record<StylePropName, string>;
 }
 
-interface ExtVersionConstraints {
+export interface ExtVersionConstraints {
   extVersion: string;
 }
 
@@ -138,12 +138,17 @@ export interface AdProviderForAllSitesRule extends ExtVersionConstraints {
   providers: string[];
 }
 
+export interface ReplaceAdsUrlsBlacklistEntry extends ExtVersionConstraints {
+  regexes: string[];
+}
+
 const AD_PLACES_RULES_KEY = 'ad_places_rules';
 const AD_PROVIDERS_BY_SITES_KEY = 'ad_providers_by_sites';
 const AD_PROVIDERS_ALL_SITES_KEY = 'ad_providers_all_sites';
 const AD_PROVIDERS_LIST_KEY = 'ad_providers_list';
 const PERMANENT_AD_PLACES_RULES_KEY = 'permanent_ad_places_rules';
 const PERMANENT_NATIVE_AD_PLACES_RULES_KEY = 'permanent_native_ad_places_rules';
+const REPLACE_ADS_URLS_BLACKLIST_KEY = 'replace_ads_urls_blacklist';
 
 export const adPlacesRulesMethods = objectStorageMethodsFactory<AdPlacesRule[]>(AD_PLACES_RULES_KEY, []);
 
@@ -164,6 +169,11 @@ export const permanentNativeAdPlacesMethods = objectStorageMethodsFactory<Perman
   []
 );
 
+export const replaceAdsUrlsBlacklistMethods = objectStorageMethodsFactory<ReplaceAdsUrlsBlacklistEntry[]>(
+  REPLACE_ADS_URLS_BLACKLIST_KEY,
+  []
+);
+
 export const getAdProvidersForAllSites = async () => redisClient.smembers(AD_PROVIDERS_ALL_SITES_KEY);
 
 export const addAdProvidersForAllSites = async (providers: string[]) =>

src/routers/slise-ad-rules/ad-places.ts

@@ -1,18 +1,27 @@
-import { Router } from 'express';
+import { Request, Router } from 'express';
 
 import {
   filterByVersion,
   permanentNativeAdPlacesMethods,
   permanentAdPlacesMethods,
-  adPlacesRulesMethods
+  adPlacesRulesMethods,
+  PermanentAdPlacesRule,
+  AdPlacesRule,
+  ExtVersionConstraints
 } from '../../advertising/external-ads';
 import { addObjectStorageMethodsToRouter } from '../../utils/express-helpers';
+import { transformValues } from '../../utils/helpers';
 import {
   hostnamesListSchema,
   permanentAdPlacesRulesDictionarySchema,
   adPlacesRulesDictionarySchema
 } from '../../utils/schemas';
 
+const transformAdPlaces = <T extends ExtVersionConstraints>(value: T[], req: Request) =>
+  filterByVersion(value, req.query.extVersion as string | undefined);
+const transformAdPlacesDictionary = <T extends ExtVersionConstraints>(rules: Record<string, T[]>, req: Request) =>
+  transformValues(rules, value => transformAdPlaces(value, req));
+
 /**
  * @swagger
  * tags:
@@ -68,13 +77,14 @@ import {
  *             type: string
  *     ExtVersionConstraints:
  *       type: object
+ *       required:
+ *         - extVersion
  *       properties:
  *         extVersion:
  *           type: string
  *           description: >
  *             A range of versions where the rule is applicable. If not specified, the rule is applicable
  *             for all versions. See the [ranges format](https://www.npmjs.com/package/semver#ranges)
- *           default: '*'
  *     AdPlacesRule:
  *       allOf:
  *         - $ref: '#/components/schemas/ExtVersionConstraints'
@@ -420,14 +430,15 @@ export const adPlacesRulesRouter = Router();
  *       '500':
  *         $ref: '#/components/responses/ErrorResponse'
  */
-addObjectStorageMethodsToRouter(adPlacesRulesRouter, {
+addObjectStorageMethodsToRouter<PermanentAdPlacesRule[]>(adPlacesRulesRouter, {
   path: '/permanent-native',
   methods: permanentNativeAdPlacesMethods,
   keyName: 'domain',
   objectValidationSchema: permanentAdPlacesRulesDictionarySchema,
   keysArrayValidationSchema: hostnamesListSchema,
   successfulRemovalMessage: entriesCount => `${entriesCount} entries have been removed`,
-  transformGotValueFn: (value, req) => filterByVersion(value, req.query.extVersion as string | undefined)
+  valueTransformFn: transformAdPlaces,
+  objectTransformFn: transformAdPlacesDictionary
 });
 
 /**
@@ -569,14 +580,15 @@ addObjectStorageMethodsToRouter(adPlacesRulesRouter, {
  *       '500':
  *         $ref: '#/components/responses/ErrorResponse'
  */
-addObjectStorageMethodsToRouter(adPlacesRulesRouter, {
+addObjectStorageMethodsToRouter<PermanentAdPlacesRule[]>(adPlacesRulesRouter, {
   path: '/permanent',
   methods: permanentAdPlacesMethods,
   keyName: 'domain',
   objectValidationSchema: permanentAdPlacesRulesDictionarySchema,
   keysArrayValidationSchema: hostnamesListSchema,
   successfulRemovalMessage: entriesCount => `${entriesCount} entries have been removed`,
-  transformGotValueFn: (value, req) => filterByVersion(value, req.query.extVersion as string | undefined)
+  valueTransformFn: transformAdPlaces,
+  objectTransformFn: transformAdPlacesDictionary
 });
 
 /**
@@ -718,12 +730,13 @@ addObjectStorageMethodsToRouter(adPlacesRulesRouter, {
  *       '500':
  *         $ref: '#/components/responses/ErrorResponse'
  */
-addObjectStorageMethodsToRouter(adPlacesRulesRouter, {
+addObjectStorageMethodsToRouter<AdPlacesRule[]>(adPlacesRulesRouter, {
   path: '/',
   methods: adPlacesRulesMethods,
   keyName: 'domain',
   objectValidationSchema: adPlacesRulesDictionarySchema,
   keysArrayValidationSchema: hostnamesListSchema,
   successfulRemovalMessage: entriesCount => `${entriesCount} entries have been removed`,
-  transformGotValueFn: (value, req) => filterByVersion(value, req.query.extVersion as string | undefined)
+  valueTransformFn: transformAdPlaces,
+  objectTransformFn: transformAdPlacesDictionary
 });

src/routers/slise-ad-rules/index.ts

@@ -2,6 +2,7 @@ import { Router } from 'express';
 
 import { adPlacesRulesRouter } from './ad-places';
 import { adProvidersRouter } from './providers';
+import { replaceUrlsBlacklistRouter } from './replace-urls-blacklist';
 
 /**
  * @swagger
@@ -41,3 +42,4 @@ export const adRulesRouter = Router();
 
 adRulesRouter.use('/ad-places', adPlacesRulesRouter);
 adRulesRouter.use('/providers', adProvidersRouter);
+adRulesRouter.use('/replace-urls-blacklist', replaceUrlsBlacklistRouter);

src/routers/slise-ad-rules/providers.ts

@@ -1,4 +1,5 @@
-import { Router } from 'express';
+import { Request, Router } from 'express';
+import { identity } from 'lodash';
 
 import {
   addAdProvidersForAllSites,
@@ -7,12 +8,14 @@ import {
   adProvidersMethods,
   adProvidersByDomainRulesMethods,
   AdProviderSelectorsRule,
-  filterByVersion
+  filterByVersion,
+  AdProvidersByDomainRule
 } from '../../advertising/external-ads';
 import { basicAuth } from '../../middlewares/basic-auth.middleware';
 import { addObjectStorageMethodsToRouter, withBodyValidation, withExceptionHandler } from '../../utils/express-helpers';
+import { transformValues } from '../../utils/helpers';
 import {
-  adTypesListSchema,
+  nonEmptyStringsListSchema,
   hostnamesListSchema,
   adProvidersByDomainsRulesDictionarySchema,
   adProvidersDictionarySchema
@@ -178,7 +181,7 @@ adProvidersRouter
   .post(
     basicAuth,
     withExceptionHandler(
-      withBodyValidation(adTypesListSchema, async (req, res) => {
+      withBodyValidation(nonEmptyStringsListSchema, async (req, res) => {
         const providersAddedCount = await addAdProvidersForAllSites(req.body);
 
         res.status(200).send({ message: `${providersAddedCount} providers have been added` });
@@ -188,7 +191,7 @@ adProvidersRouter
   .delete(
     basicAuth,
     withExceptionHandler(
-      withBodyValidation(adTypesListSchema, async (req, res) => {
+      withBodyValidation(nonEmptyStringsListSchema, async (req, res) => {
         const providersRemovedCount = await removeAdProvidersForAllSites(req.body);
 
         res.status(200).send({ message: `${providersRemovedCount} providers have been removed` });
@@ -287,13 +290,15 @@ adProvidersRouter
  *       '500':
  *         $ref: '#/components/responses/ErrorResponse'
  */
-addObjectStorageMethodsToRouter(adProvidersRouter, {
+addObjectStorageMethodsToRouter<AdProvidersByDomainRule[]>(adProvidersRouter, {
   path: '/by-sites',
   methods: adProvidersByDomainRulesMethods,
   keyName: 'domain',
   objectValidationSchema: adProvidersByDomainsRulesDictionarySchema,
   keysArrayValidationSchema: hostnamesListSchema,
-  successfulRemovalMessage: entriesCount => `${entriesCount} entries have been removed`
+  successfulRemovalMessage: entriesCount => `${entriesCount} entries have been removed`,
+  valueTransformFn: identity,
+  objectTransformFn: identity
 });
 
 /**
@@ -437,19 +442,22 @@ addObjectStorageMethodsToRouter(adProvidersRouter, {
  *       '500':
  *         $ref: '#/components/responses/ErrorResponse'
  */
-addObjectStorageMethodsToRouter<AdProviderSelectorsRule[], string[]>(adProvidersRouter, {
+const transformAdProviderSelectorsRules = (rules: AdProviderSelectorsRule[], req: Request) =>
+  Array.from(
+    new Set(
+      filterByVersion(rules, req.query.extVersion as string | undefined)
+        .map(({ selectors }) => selectors)
+        .flat()
+    )
+  );
+
+addObjectStorageMethodsToRouter<AdProviderSelectorsRule[], Record<string, string[]>, string[]>(adProvidersRouter, {
   path: '/',
   methods: adProvidersMethods,
   keyName: 'providerId',
   objectValidationSchema: adProvidersDictionarySchema,
-  keysArrayValidationSchema: adTypesListSchema,
+  keysArrayValidationSchema: nonEmptyStringsListSchema,
   successfulRemovalMessage: entriesCount => `${entriesCount} providers have been removed`,
-  transformGotValueFn: (rules, req) =>
-    Array.from(
-      new Set(
-        filterByVersion(rules, req.query.extVersion as string | undefined)
-          .map(({ selectors }) => selectors)
-          .flat()
-      )
-    )
+  valueTransformFn: transformAdProviderSelectorsRules,
+  objectTransformFn: (rules, req) => transformValues(rules, value => transformAdProviderSelectorsRules(value, req))
 });