Merge pull request #2 from KaanSK/develop

Develop - V2.0

.github/workflows/build.yml

@@ -0,0 +1,24 @@
+name: build
+
+on:
+  pull_request:
+  push:
+
+permissions:
+  contents: read
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-go@v3
+        with:
+          go-version: 1.18
+      - uses: actions/cache@v3
+        with:
+          path: ~/go/pkg/mod
+          key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
+          restore-keys: |
+            ${{ runner.os }}-go-
+      - run: go test -v ./...

.github/workflows/release.yml

@@ -0,0 +1,54 @@
+#
+# Releaser workflow setup
+# https://goreleaser.com/ci/actions/
+#
+name: release
+
+# run only on tags
+on:
+  push:
+    tags:
+      - 'v*'
+
+permissions:
+   contents: write # needed to write releases
+   id-token: write # needed for keyless signing
+   packages: write # needed for ghcr access
+
+jobs:
+  release:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0 # this is important, otherwise it won't checkout the full tree (i.e. no previous tags)
+      - uses: actions/setup-go@v3
+        with:
+          go-version: 1.18
+      - uses: actions/cache@v3
+        with:
+          path: ~/go/pkg/mod
+          key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
+          restore-keys: |
+            ${{ runner.os }}-go-
+      - uses: sigstore/cosign-installer@v2.4.0         # installs cosign
+      - uses: anchore/sbom-action/download-syft@v0.11.0 # installs syft
+
+      - name: dockerhub login
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+
+      - name: ghcr login
+        uses: docker/login-action@v2                   # login to ghcr
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - uses: goreleaser/goreleaser-action@v3          # run goreleaser
+        with:
+          version: latest
+          args: release --rm-dist
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.goreleaser.Dockerfile

@@ -0,0 +1,3 @@
+FROM scratch
+COPY shomon /usr/local/bin/shomon
+ENTRYPOINT [ "/usr/local/bin/shomon" ]

.goreleaser.yaml

@@ -0,0 +1,84 @@
+builds:
+- env:
+  - CGO_ENABLED=0
+  goos:
+  - linux
+  - darwin
+  goarch:
+  - amd64
+  - arm64
+  # ensures mod timestamp to be the commit timestamp
+  mod_timestamp: '{{ .CommitTimestamp }}'
+  binary: shomon
+  flags:
+    # trims path
+    - -trimpath
+  ldflags:
+    # use commit date instead of current date as main.date
+    # only needed if you actually use those things in your main package, otherwise can be ignored.
+    - -s -w -X main.version={{.Version}} -X main.commit={{.Commit}} -X main.date={{ .CommitDate }}
+
+# proxies from the go mod proxy before building
+# https://goreleaser.com/customization/gomod
+#gomod:
+#  proxy: true
+
+# config the checksum filename
+# https://goreleaser.com/customization/checksum
+checksum:
+  name_template: 'checksums.txt'
+
+# create a source tarball
+# https://goreleaser.com/customization/source/
+source:
+  enabled: true
+
+# creates SBOMs of all archives and the source tarball using syft
+# https://goreleaser.com/customization/sbom
+sboms:
+  - artifacts: archive
+  - id: source # Two different sbom configurations need two different IDs
+    artifacts: source
+
+# signs the checksum file
+# all files (including the sboms) are included in the checksum, so we don't need to sign each one if we don't want to
+# https://goreleaser.com/customization/sign
+signs:
+- cmd: cosign
+  env:
+  - COSIGN_EXPERIMENTAL=1
+  certificate: '${artifact}.pem'
+  args:
+    - sign-blob
+    - '--output-certificate=${certificate}'
+    - '--output-signature=${signature}'
+    - '${artifact}'
+  artifacts: checksum
+  output: true
+
+
+dockers:
+- image_templates:
+  - "docker.io/kaansk/shomon:{{ .Tag }}"
+  - "docker.io/kaansk/shomon:latest"
+  - "ghcr.io/kaansk/shomon:{{ .Tag }}"
+  - "ghcr.io/kaansk/shomon:latest"
+  dockerfile: .goreleaser.Dockerfile
+  build_flag_templates:
+  - "--pull"
+  - "--label=org.opencontainers.image.created={{.Date}}"
+  - "--label=org.opencontainers.image.name={{.ProjectName}}"
+  - "--label=org.opencontainers.image.revision={{.FullCommit}}"
+  - "--label=org.opencontainers.image.version={{.Version}}"
+  - "--label=org.opencontainers.image.source={{.GitURL}}"
+
+
+docker_signs:
+  - cmd: cosign
+    env:
+    - COSIGN_EXPERIMENTAL=1
+    artifacts: images
+    output: true
+    args:
+    - 'sign'
+    - '${artifact}'

Dockerfile

@@ -1,6 +1,6 @@
 # Taken from https://github.com/chemidy/smallest-secured-golang-docker-image
 
-FROM golang@sha256:244a736db4a1d2611d257e7403c729663ce2eb08d4628868f9d9ef2735496659 as builder
+FROM golang:alpine as builder
 
 # Install git + SSL ca certificates.
 # Git is required for fetching the dependencies.

README.md

@@ -1,32 +1,53 @@
-# [![ShoMon](./images/logo.png)]()
-
-ShoMon is a shodan alert feeder for theHive written in GoLang. Takes advantage of Golang's goroutines with deferred recover to continuously monitor alerts from shodan and feed them into theHive as alerts.
+<p align="center">
+  <img src="images/logo.png" />
+</p>
+<p align="center">
+ShoMon is a Shodan alert feeder for TheHive written in GoLang. With version 2.0, it is more powerful than ever!
+</p>
+
+
+# Functionalities
+* Can be used as Webhook OR Stream listener
+    * Webhook listener opens a restful API endpoint for Shodan to send alerts. This means you need to make this endpoint available to public net
+    * Stream listener connects to Shodan and fetches/parses the alert stream
+* Utilizes [shadowscatcher/shodan](https://github.com/shadowscatcher/shodan) (fantastic work) for Shodan interaction.
+* Alert specifics can be adjusted via conf.yaml or environment variables
+* Console logs are in JSON format and can be ingested by any other further log management tools
+* CI/CD via Github Actions ensures that a proper Release with changelogs, artifacts, images on ghcr and dockerhub will be provided
+* Provides a working [docker-compose file](docker-compose.yml) file for TheHive, dependencies
+* Super fast and Super mini in size
+* Complete code refactoring in v2.0 resulted in more modular, maintainable code
 
 # Usage
-   [![Help](./images/help1.png)]()
+* Parameters should be provided via ```conf.yaml``` or environment variables. Please see [config file](conf.yaml) and [docker-compose file](docker-compose.yml)
+* After conf or environment variables are set simply issue command: 
+
+    `./shomon`
 
 ## Notes
-* Logs can be found in shodanmonitor.log under the same folder
-* Alert reference is md5("ip:port")
-* Default logging level is DEBUG. Can be changed via editing logwrapper
+* Alert reference is first 6 chars of md5("ip:port")
+* Only 1 mod can be active at a time. Webhook and Stream listener can not be activated together.
 
 # Setup & Compile Instructions
 ## Get latest compiled binary from releases
-1. Check [Releases] section.
+1. Check [Releases](https://github.com/KaanSK/shomon/releases/latest)  section.
 
 ## Compile from source code
 1. Make sure that you have a working Golang workspace.
 2. `go build .`
     * `go build -ldflags="-s -w" .` could be used to customize compilation and produce smaller binary.
 
-## Using Dockerfile
-1. `docker build -t shomon .`
-2. `docker run -it shomon -s {SHODANKEY} -t {THEHIVEKEY}`
+## Using [Dockerfile](Dockerfile)
+1. Edit [config file](conf.yaml) or provide environment variables to commands bellow
+2. `docker build -t shomon .`
+3. `docker run -it shomon`
+
+## Using [docker-compose file](docker-compose.yml)
+1. Edit environment variables and configurations in [docker-compose file](docker-compose.yml)
+2. `docker-compose run -d`
 
 # Credits
 * Logo Made via LogoMakr.com
-* `go-shodan` : https://github.com/ns3777k/go-shodan
-* logwrapper package : https://www.datadoghq.com/blog/go-logging/
-* Dockerfile : https://www.cloudreach.com/en/resources/blog/cts-build-golang-dockerfiles/
-
-[Releases]: https://github.com/KaanSK/shomon/releases/latest
+* [shadowscatcher/shodan](https://github.com/shadowscatcher/shodan) 
+* [Dockerfile Reference](https://www.cloudreach.com/en/resources/blog/cts-build-golang-dockerfiles/) 
+* Release management with [GoReleaser](https://goreleaser.com)

conf.yaml

@@ -0,0 +1,13 @@
+HIVE_URL: "http://localhost:9000"
+HIVE_KEY:
+HIVE_CASE_TEMPLATE:
+HIVE_TYPE: "SHODAN"
+HIVE_TAGS:
+  - 'test:test2="test3"'
+  - 'test4:test5="test6"'
+SHODAN_KEY:
+INCLUDE_BANNER: true
+LOG_LEVEL: DEBUG
+WEBHOOK: true
+WEBHOOK_ENDPOINT: "/"
+WEBHOOK_PORT: 8080

conf/conf.go

@@ -0,0 +1,67 @@
+package conf
+
+import (
+	"errors"
+	"fmt"
+	"os"
+	"strings"
+
+	"github.com/knadh/koanf"
+	"github.com/knadh/koanf/parsers/yaml"
+	"github.com/knadh/koanf/providers/confmap"
+	"github.com/knadh/koanf/providers/env"
+	"github.com/knadh/koanf/providers/file"
+)
+
+type ShomonConfig struct {
+	HiveUrl          string   `koanf:"HIVE_URL"`
+	HiveCaseTemplate string   `koanf:"HIVE_CASE_TEMPLATE" `
+	HiveKey          string   `koanf:"HIVE_KEY"`
+	HiveTags         []string `koanf:"HIVE_TAGS"`
+	HiveType         string   `koanf:"HIVE_TYPE"`
+	ShodanKey        string   `koanf:"SHODAN_KEY"`
+	LogLevel         string   `koanf:"LOG_LEVEL"`
+	IncludeBanner    bool     `koanf:"INCLUDE_BANNER"`
+	Webhook          bool     `koanf:"WEBHOOK"`
+	WebhookEndpoint  string   `koanf:"WEBHOOK_ENDPOINT"`
+	WebhookPort      int      `koanf:"WEBHOOK_PORT"`
+}
+
+func New() (conf ShomonConfig, err error) {
+	newConfig := &ShomonConfig{}
+	if err := newConfig.Setup(); err != nil {
+		return *newConfig, err
+	}
+	return *newConfig, nil
+}
+
+func (d *ShomonConfig) Setup() error {
+	k := koanf.New(".")
+
+	//Default Values
+	k.Load(confmap.Provider(map[string]interface{}{
+		"HIVE_URL":  "http://localhost:9000",
+		"HIVE_KEY":  "NO_KEY",
+		"LOG_LEVEL": "INFO",
+	}, "."), nil)
+
+	if err := k.Load(file.Provider("conf.yaml"), yaml.Parser()); err != nil {
+		if errors.Is(err, os.ErrNotExist) {
+			k.Load(env.Provider("SHOMON_", ".", func(s string) string {
+				return strings.TrimPrefix(s, "SHOMON_")
+			}), nil)
+		} else {
+			return err
+		}
+	}
+
+	k.Unmarshal("", &d)
+	return nil
+}
+
+func (d *ShomonConfig) Print() string {
+	if d != nil {
+		return fmt.Sprintf("%+v", d)
+	}
+	return ""
+}

conf/conf_test.go

@@ -0,0 +1,15 @@
+package conf
+
+import (
+	"testing"
+)
+
+func TestGetConf(t *testing.T) {
+	conf, err := New()
+	if err != nil {
+		t.Errorf(err.Error())
+	}
+	if conf.LogLevel == "" {
+		t.Errorf("Config could not be populated")
+	}
+}