Merge branch 'release/0.1'

.env.default

@@ -0,0 +1,13 @@
+COMMON_APP_ENV=production
+COMMON_DEV_TOOL_URL=http://chromedp:9222
+URLSCAN_API_KEY=<Replace here to the actual API key>
+URLSCAN_API_URL=https://urlscan.io/api
+GOOGLE_SAFE_BROWSING_API_KEY=<Replace here to the actual API key>
+GOOGLE_SAFE_BROWSING_API_URL=https://safebrowsing.googleapis.com/v4/threatMatches:find
+GOOGLE_TRANSPARENCYREPORT_API_URL=https://transparencyreport.google.com/transparencyreport/api/v3/safebrowsing/
+# These are used for requesting to external APIs.
+COMMON_MAX_IDLE_CONNS=200
+COMMON_MAX_IDLE_CONN_SPER_HOST=200
+COMMON_MAX_CONNS_PER_HOST=200
+COMMON_IDLE_CONN_TIMEOUT=60
+COMMON_DISABLE_COMPRESSION=true

.realize.yaml

@@ -0,0 +1,23 @@
+settings:
+  legacy:
+    force: false
+    interval: 0s
+schema:
+- name: studio-abuse-detector
+  path: .
+  commands:
+    install:
+      status: true
+      method: go build -o app
+    run:
+      status: true
+      method: ./app
+  watcher:
+    extensions:
+    - go
+    paths:
+    - /
+    ignored_paths:
+    - .git
+    - .realize
+    - vendor

Dockerfile

@@ -0,0 +1,27 @@
+FROM golang:1.14.9-alpine3.12 as build
+
+WORKDIR /go/app
+
+COPY . .
+COPY .env .
+
+RUN apk add --no-cache git \
+ && go build -o app
+
+FROM alpine:3.12.0
+
+WORKDIR /app
+
+COPY --from=build /go/app/app .
+
+RUN apk add --update --no-cache go git \
+  && export GOPATH=/root/go \
+  && export PATH=${GOPATH}/bin:/usr/local/go/bin:$PATH \
+  && export GOBIN=$GOROOT/bin \
+  && mkdir -p ${GOPATH}/src ${GOPATH}/bin \
+  && addgroup go \
+  && adduser -D -G go go \
+  && chown -R go:go /app/app \
+  && chmod +x /app/app
+
+CMD ["go", "run", "main.go"]

DockerfileChrome

@@ -0,0 +1,3 @@
+FROM zenika/alpine-chrome
+
+CMD ["--no-sandbox", "--remote-debugging-address=0.0.0.0", "--remote-debugging-port=9222"]

README.md

@@ -1,16 +1,102 @@
 # Abuse Detector
-This application is for previnting phishing sites are created on Studio.
+This application is for preventing phishing sites are created on Studio. 
 
 ## Requirements
 - Go 1.14.9 >=
-- Docker
-- Docker Compose
+- Docker 2.4.0.0 >=
+- Docker Compose 1.27.4 >=
 
+## Usage
+### Request verification
+This API verifies if the site does not include malicious links, such as fishing.
+```
+http://localhost:3000/verify?url=https://www.google.com/
+```
+The response would look like below if the site is not malicious.
+```
+{
+    "strategyName": "",
+    "link": [],
+    "malicious": false,
+    "statusCode": 200,
+    "error": null
+}
+```
+If it's malicious, the response looks like below.
+```
+{
+    "strategyName": "TransparencyReportVerifyStrategy",
+    "link": ["http://sucursalvirtualpersonas-sa.com"],
+    "malicious": true,
+    "statusCode": 200,
+    "error": null
+}
+```
 ## How to build
 ```shell script
-go build
+go build *.go
 ```    
-## How to run
-```shell script
-go run main.go
-```
+## How to run for Development
+1. Create `.env` based off from `.env.default`. For API keys required, please refer documents below in this README. 
+1. In `.env` file, Remove `production` string from `COMMON_APP_ENV` as follows.
+    ```
+    COMMON_APP_ENV=
+    ```
+1. Start Chrome Headless Server
+    ```
+    docker run -d -p 9222:9222 --rm --name headless-shell --shm-size 2G chromedp/headless-shell
+    ```
+1. Run server as below. `realize` command allows Hot reloading.
+    ```shell script
+    realize start
+    ```
+
+## How to run all tests
+```
+go test -v -race -run=. -bench=. ./...
+```
+
+##How to run for production
+1. Create `.env` based off from `.env.default`
+1. Set API Keys accordingly.
+1. Run command below.
+    ```
+    docker-compose up
+    ```
+
+## How to build Docker image
+This is how to build and confirm the image is built correctly.
+```
+docker build -t studio-abuse-detector .
+docker run -p 3000:3000 -d --name studio-abuse-detector studio-abuse-detector:latest
+curl localhost:3000
+```
+
+## Opearation Related
+### How to remove all images including running
+```~~~~
+docker rm -f `docker ps -qa`
+```
+### How to access an image
+```
+docker-compose exec app /bin/sh
+```
+
+## Appendix
+- [cdp, Chrome Dev Tools Protocl](https://github.com/mafredri/cdp)
+- [Headless Chrome server base for Dockerfile, Zenika/alpine-chrome](https://github.com/Zenika/alpine-chrome)
+
+### How to get API key for urlscan.io
+1. Go to `https://urlscan.io/` and create an account.
+1. Go to [Settings & API](https://urlscan.io/user/profile/) and create an API Key
+1. Copy the `Key` and set it to `URLSCAN_API_KEY` in the `.env` file
+
+### How to get API key for Google Safe Browsing API
+1. Access to [Google API Console](https://console.developers.google.com/) and create a project
+1. Create API key in the project.
+1. Look for `Google Safe Browsing API` in `Liberary` tab and add it for the API Key created.
+1. Copy the `Key` and set it to `GOOGLE_SAFE_BROWSING_API_KEY` in the `.env` file
+
+## Caveat
+- Chrome Headless server in use may need load balancing for a more massive load of access.
+- Test links are real phishing sites for now. They become offline or removed in the short term, so tests highly likely to fail.

docker-compose.yml

@@ -0,0 +1,16 @@
+version: "3"
+services:
+  chromedp:
+    build:
+      context: .
+      dockerfile: DockerfileChrome
+    ports:
+      - 9222:9222
+  app:
+    build:
+      context: .
+      dockerfile: Dockerfile
+    ports:
+      - 3000:3000
+    volumes:
+      - ./:/app

go.mod

@@ -4,7 +4,18 @@ go 1.14
 
 require (
 	github.com/PuerkitoBio/goquery v1.5.1
+	github.com/chromedp/cdproto v0.0.0-20200116234248-4da64dd111ac
+	github.com/chromedp/chromedp v0.5.3
 	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/go-resty/resty/v2 v2.3.0
+	github.com/joho/godotenv v1.3.0
+	github.com/kelseyhightower/envconfig v1.4.0
 	github.com/kr/pretty v0.2.1
 	github.com/labstack/echo/v4 v4.1.17
+	github.com/mafredri/cdp v0.29.2
+	github.com/op/go-logging v0.0.0-20160315200505-970db520ece7
+	github.com/pkg/errors v0.8.1
+	github.com/stretchr/testify v1.4.0
+	github.com/thoas/go-funk v0.7.0
+	gopkg.in/yaml.v2 v2.3.0 // indirect
 )