diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 0000000..0a33a54 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,24 @@ +name: build + +on: + pull_request: + push: + +permissions: + contents: read + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-go@v3 + with: + go-version: 1.18 + - uses: actions/cache@v3 + with: + path: ~/go/pkg/mod + key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-go- + - run: go test -v ./... \ No newline at end of file diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..3cf9508 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,54 @@ +# +# Releaser workflow setup +# https://goreleaser.com/ci/actions/ +# +name: release + +# run only on tags +on: + push: + tags: + - 'v*' + +permissions: + contents: write # needed to write releases + id-token: write # needed for keyless signing + packages: write # needed for ghcr access + +jobs: + release: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 # this is important, otherwise it won't checkout the full tree (i.e. no previous tags) + - uses: actions/setup-go@v3 + with: + go-version: 1.18 + - uses: actions/cache@v3 + with: + path: ~/go/pkg/mod + key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-go- + - uses: sigstore/cosign-installer@v2.4.0 # installs cosign + - uses: anchore/sbom-action/download-syft@v0.11.0 # installs syft + + - name: dockerhub login + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + + - name: ghcr login + uses: docker/login-action@v2 # login to ghcr + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + - uses: goreleaser/goreleaser-action@v3 # run goreleaser + with: + version: latest + args: release --rm-dist + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} \ No newline at end of file diff --git a/.goreleaser.Dockerfile b/.goreleaser.Dockerfile new file mode 100644 index 0000000..b4ed9e5 --- /dev/null +++ b/.goreleaser.Dockerfile @@ -0,0 +1,3 @@ +FROM scratch +COPY shomon /usr/local/bin/shomon +ENTRYPOINT [ "/usr/local/bin/shomon" ] \ No newline at end of file diff --git a/.goreleaser.yaml b/.goreleaser.yaml new file mode 100644 index 0000000..103b84d --- /dev/null +++ b/.goreleaser.yaml @@ -0,0 +1,84 @@ +builds: +- env: + - CGO_ENABLED=0 + goos: + - linux + - darwin + goarch: + - amd64 + - arm64 + # ensures mod timestamp to be the commit timestamp + mod_timestamp: '{{ .CommitTimestamp }}' + binary: shomon + flags: + # trims path + - -trimpath + ldflags: + # use commit date instead of current date as main.date + # only needed if you actually use those things in your main package, otherwise can be ignored. + - -s -w -X main.version={{.Version}} -X main.commit={{.Commit}} -X main.date={{ .CommitDate }} + +# proxies from the go mod proxy before building +# https://goreleaser.com/customization/gomod +#gomod: +# proxy: true + +# config the checksum filename +# https://goreleaser.com/customization/checksum +checksum: + name_template: 'checksums.txt' + +# create a source tarball +# https://goreleaser.com/customization/source/ +source: + enabled: true + +# creates SBOMs of all archives and the source tarball using syft +# https://goreleaser.com/customization/sbom +sboms: + - artifacts: archive + - id: source # Two different sbom configurations need two different IDs + artifacts: source + +# signs the checksum file +# all files (including the sboms) are included in the checksum, so we don't need to sign each one if we don't want to +# https://goreleaser.com/customization/sign +signs: +- cmd: cosign + env: + - COSIGN_EXPERIMENTAL=1 + certificate: '${artifact}.pem' + args: + - sign-blob + - '--output-certificate=${certificate}' + - '--output-signature=${signature}' + - '${artifact}' + artifacts: checksum + output: true + + +dockers: +- image_templates: + - "docker.io/kaansk/shomon:{{ .Tag }}" + - "docker.io/kaansk/shomon:latest" + - "ghcr.io/kaansk/shomon:{{ .Tag }}" + - "ghcr.io/kaansk/shomon:latest" + dockerfile: .goreleaser.Dockerfile + build_flag_templates: + - "--pull" + - "--label=org.opencontainers.image.created={{.Date}}" + - "--label=org.opencontainers.image.name={{.ProjectName}}" + - "--label=org.opencontainers.image.revision={{.FullCommit}}" + - "--label=org.opencontainers.image.version={{.Version}}" + - "--label=org.opencontainers.image.source={{.GitURL}}" + + +docker_signs: + - cmd: cosign + env: + - COSIGN_EXPERIMENTAL=1 + artifacts: images + output: true + args: + - 'sign' + - '${artifact}' \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index f48b425..7c91f0f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ # Taken from https://github.com/chemidy/smallest-secured-golang-docker-image -FROM golang@sha256:244a736db4a1d2611d257e7403c729663ce2eb08d4628868f9d9ef2735496659 as builder +FROM golang:alpine as builder # Install git + SSL ca certificates. # Git is required for fetching the dependencies. diff --git a/README.md b/README.md index a94ceed..2861585 100644 --- a/README.md +++ b/README.md @@ -1,32 +1,53 @@ -# [![ShoMon](./images/logo.png)]() - -ShoMon is a shodan alert feeder for theHive written in GoLang. Takes advantage of Golang's goroutines with deferred recover to continuously monitor alerts from shodan and feed them into theHive as alerts. +

+ +

+

+ShoMon is a Shodan alert feeder for TheHive written in GoLang. With version 2.0, it is more powerful than ever! +

+ + +# Functionalities +* Can be used as Webhook OR Stream listener + * Webhook listener opens a restful API endpoint for Shodan to send alerts. This means you need to make this endpoint available to public net + * Stream listener connects to Shodan and fetches/parses the alert stream +* Utilizes [shadowscatcher/shodan](https://github.com/shadowscatcher/shodan) (fantastic work) for Shodan interaction. +* Alert specifics can be adjusted via conf.yaml or environment variables +* Console logs are in JSON format and can be ingested by any other further log management tools +* CI/CD via Github Actions ensures that a proper Release with changelogs, artifacts, images on ghcr and dockerhub will be provided +* Provides a working [docker-compose file](docker-compose.yml) file for TheHive, dependencies +* Super fast and Super mini in size +* Complete code refactoring in v2.0 resulted in more modular, maintainable code # Usage - [![Help](./images/help1.png)]() +* Parameters should be provided via ```conf.yaml``` or environment variables. Please see [config file](conf.yaml) and [docker-compose file](docker-compose.yml) +* After conf or environment variables are set simply issue command: + + `./shomon` ## Notes -* Logs can be found in shodanmonitor.log under the same folder -* Alert reference is md5("ip:port") -* Default logging level is DEBUG. Can be changed via editing logwrapper +* Alert reference is first 6 chars of md5("ip:port") +* Only 1 mod can be active at a time. Webhook and Stream listener can not be activated together. # Setup & Compile Instructions ## Get latest compiled binary from releases -1. Check [Releases] section. +1. Check [Releases](https://github.com/KaanSK/shomon/releases/latest) section. ## Compile from source code 1. Make sure that you have a working Golang workspace. 2. `go build .` * `go build -ldflags="-s -w" .` could be used to customize compilation and produce smaller binary. -## Using Dockerfile -1. `docker build -t shomon .` -2. `docker run -it shomon -s {SHODANKEY} -t {THEHIVEKEY}` +## Using [Dockerfile](Dockerfile) +1. Edit [config file](conf.yaml) or provide environment variables to commands bellow +2. `docker build -t shomon .` +3. `docker run -it shomon` + +## Using [docker-compose file](docker-compose.yml) +1. Edit environment variables and configurations in [docker-compose file](docker-compose.yml) +2. `docker-compose run -d` # Credits * Logo Made via LogoMakr.com -* `go-shodan` : https://github.com/ns3777k/go-shodan -* logwrapper package : https://www.datadoghq.com/blog/go-logging/ -* Dockerfile : https://www.cloudreach.com/en/resources/blog/cts-build-golang-dockerfiles/ - -[Releases]: https://github.com/KaanSK/shomon/releases/latest +* [shadowscatcher/shodan](https://github.com/shadowscatcher/shodan) +* [Dockerfile Reference](https://www.cloudreach.com/en/resources/blog/cts-build-golang-dockerfiles/) +* Release management with [GoReleaser](https://goreleaser.com) diff --git a/conf.yaml b/conf.yaml new file mode 100644 index 0000000..daf0713 --- /dev/null +++ b/conf.yaml @@ -0,0 +1,13 @@ +HIVE_URL: "http://localhost:9000" +HIVE_KEY: +HIVE_CASE_TEMPLATE: +HIVE_TYPE: "SHODAN" +HIVE_TAGS: + - 'test:test2="test3"' + - 'test4:test5="test6"' +SHODAN_KEY: +INCLUDE_BANNER: true +LOG_LEVEL: DEBUG +WEBHOOK: true +WEBHOOK_ENDPOINT: "/" +WEBHOOK_PORT: 8080 \ No newline at end of file diff --git a/conf/conf.go b/conf/conf.go new file mode 100644 index 0000000..d4f2ae9 --- /dev/null +++ b/conf/conf.go @@ -0,0 +1,67 @@ +package conf + +import ( + "errors" + "fmt" + "os" + "strings" + + "github.com/knadh/koanf" + "github.com/knadh/koanf/parsers/yaml" + "github.com/knadh/koanf/providers/confmap" + "github.com/knadh/koanf/providers/env" + "github.com/knadh/koanf/providers/file" +) + +type ShomonConfig struct { + HiveUrl string `koanf:"HIVE_URL"` + HiveCaseTemplate string `koanf:"HIVE_CASE_TEMPLATE" ` + HiveKey string `koanf:"HIVE_KEY"` + HiveTags []string `koanf:"HIVE_TAGS"` + HiveType string `koanf:"HIVE_TYPE"` + ShodanKey string `koanf:"SHODAN_KEY"` + LogLevel string `koanf:"LOG_LEVEL"` + IncludeBanner bool `koanf:"INCLUDE_BANNER"` + Webhook bool `koanf:"WEBHOOK"` + WebhookEndpoint string `koanf:"WEBHOOK_ENDPOINT"` + WebhookPort int `koanf:"WEBHOOK_PORT"` +} + +func New() (conf ShomonConfig, err error) { + newConfig := &ShomonConfig{} + if err := newConfig.Setup(); err != nil { + return *newConfig, err + } + return *newConfig, nil +} + +func (d *ShomonConfig) Setup() error { + k := koanf.New(".") + + //Default Values + k.Load(confmap.Provider(map[string]interface{}{ + "HIVE_URL": "http://localhost:9000", + "HIVE_KEY": "NO_KEY", + "LOG_LEVEL": "INFO", + }, "."), nil) + + if err := k.Load(file.Provider("conf.yaml"), yaml.Parser()); err != nil { + if errors.Is(err, os.ErrNotExist) { + k.Load(env.Provider("SHOMON_", ".", func(s string) string { + return strings.TrimPrefix(s, "SHOMON_") + }), nil) + } else { + return err + } + } + + k.Unmarshal("", &d) + return nil +} + +func (d *ShomonConfig) Print() string { + if d != nil { + return fmt.Sprintf("%+v", d) + } + return "" +} diff --git a/conf/conf_test.go b/conf/conf_test.go new file mode 100644 index 0000000..1ee6809 --- /dev/null +++ b/conf/conf_test.go @@ -0,0 +1,15 @@ +package conf + +import ( + "testing" +) + +func TestGetConf(t *testing.T) { + conf, err := New() + if err != nil { + t.Errorf(err.Error()) + } + if conf.LogLevel == "" { + t.Errorf("Config could not be populated") + } +} diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..3793e86 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,81 @@ +version: "3" +services: + shomon: + build: . + container_name: shomon + ports: + - "8080:8080" + environment: + - "SHOMON_HIVE_URL=http://thehive:9000" + - "SHOMON_HIVE_KEY=" + - "SHOMON_HIVE_TYPE=SHODAN" + - 'SHOMON_HIVE_TAGS=test:test2="test3"' + - "SHOMON_SHODAN_KEY=" + - "SHOMON_INCLUDE_BANNER=true" + - "SHOMON_LOG_LEVEL=DEBUG" + - "SHOMON_WEBHOOK=true" + - "SHOMON_WEBHOOK_ENDPOINT=/banner" + - "SHOMON_WEBHOOK_PORT=8080" + + thehive: + image: strangebee/thehive:latest + container_name: thehive + depends_on: + - cassandra + - elasticsearch + - minio + mem_limit: 1000m + ports: + - "9000:9000" + environment: + - JVM_OPTS="-Xms1024M -Xmx1024M" + command: + - --secret + - "mySecretForTheHive" + - "--cql-hostnames" + - "cassandra" + - "--cql-username" + - "cassandra" + - "--cql-password" + - "cassandra" + - "--index-backend" + - "elasticsearch" + - "--es-hostnames" + - "elasticsearch" + - "--s3-endpoint" + - "http://minio:9000" + - "--s3-access-key" + - "minioadmin" + - "--s3-secret-key" + - "minioadmin" + - "--s3-use-path-access-style" + - "--no-config-cortex" + + cassandra: + container_name: cassandra + image: bitnami/cassandra + ports: + - "9042:9042" + environment: + - CASSANDRA_CLUSTER_NAME=TheHive + + elasticsearch: + container_name: elastic + mem_limit: 1000m + image: docker.elastic.co/elasticsearch/elasticsearch:7.16.2 + ports: + - "9200:9200" + environment: + - discovery.type=single-node + - xpack.security.enabled=false + + minio: + container_name: minio + mem_limit: 1000m + image: quay.io/minio/minio + command: ["minio", "server", "/data", "--console-address", ":9001"] + environment: + - MINIO_ROOT_USER=minioadmin + - MINIO_ROOT_PASSWORD=minioadmin + ports: + - "9001:9001" diff --git a/go.mod b/go.mod index 84c2beb..605aae2 100644 --- a/go.mod +++ b/go.mod @@ -1,9 +1,21 @@ module github.com/KaanSK/shomon -go 1.14 +go 1.18 require ( - github.com/jessevdk/go-flags v1.4.0 + github.com/jarcoal/httpmock v1.2.0 + github.com/knadh/koanf v1.4.2 + github.com/shadowscatcher/shodan v1.0.6 + go.uber.org/zap v1.21.0 +) - github.com/sirupsen/logrus v1.6.0 +require ( + github.com/fsnotify/fsnotify v1.4.9 // indirect + github.com/mitchellh/copystructure v1.2.0 // indirect + github.com/mitchellh/mapstructure v1.4.1 // indirect + github.com/mitchellh/reflectwalk v1.0.2 // indirect + go.uber.org/atomic v1.7.0 // indirect + go.uber.org/multierr v1.6.0 // indirect + golang.org/x/sys v0.0.0-20220615213510-4f61da869c0c // indirect + gopkg.in/yaml.v3 v3.0.0 // indirect ) diff --git a/go.sum b/go.sum index 001af06..136d8ad 100644 --- a/go.sum +++ b/go.sum @@ -1,35 +1,190 @@ -github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= -github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e h1:fY5BOSpyZCqRo5OhCuC+XN+r/bBCmeuuJtjz+bCNIf8= -github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= -github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= +github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= +github.com/aws/aws-sdk-go-v2 v1.9.2/go.mod h1:cK/D0BBs0b/oWPIcX/Z/obahJK1TT7IPVjy53i/mX/4= +github.com/aws/aws-sdk-go-v2/config v1.8.3/go.mod h1:4AEiLtAb8kLs7vgw2ZV3p2VZ1+hBavOc84hqxVNpCyw= +github.com/aws/aws-sdk-go-v2/credentials v1.4.3/go.mod h1:FNNC6nQZQUuyhq5aE5c7ata8o9e4ECGmS4lAXC7o1mQ= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.6.0/go.mod h1:gqlclDEZp4aqJOancXK6TN24aKhT0W0Ae9MHk3wzTMM= +github.com/aws/aws-sdk-go-v2/internal/ini v1.2.4/go.mod h1:ZcBrrI3zBKlhGFNYWvju0I3TR93I7YIgAfy82Fh4lcQ= +github.com/aws/aws-sdk-go-v2/service/appconfig v1.4.2/go.mod h1:FZ3HkCe+b10uFZZkFdvf98LHW21k49W8o8J366lqVKY= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.3.2/go.mod h1:72HRZDLMtmVQiLG2tLfQcaWLCssELvGl+Zf2WVxMmR8= +github.com/aws/aws-sdk-go-v2/service/sso v1.4.2/go.mod h1:NBvT9R1MEF+Ud6ApJKM0G+IkPchKS7p7c2YPKwHmBOk= +github.com/aws/aws-sdk-go-v2/service/sts v1.7.2/go.mod h1:8EzeIqfWt2wWT4rJVu3f21TfrhJ8AEMzVybRNSb/b4g= +github.com/aws/smithy-go v1.8.0/go.mod h1:SObp3lf9smib00L/v3U2eAKG8FyQ7iLrJnQiAmR5n+E= +github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= +github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= +github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk= -github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= -github.com/google/pprof v0.0.0-20200507031123-427632fa3b1c h1:lIC98ZUNah83ky7d9EXktLFe4H7Nwus59dTOLXr8xAI= -github.com/google/pprof v0.0.0-20200507031123-427632fa3b1c/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6 h1:UDMh68UUwekSh5iP2OMhRRZJiiBccgV7axzUG8vi56c= -github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/ianlancetaylor/demangle v0.0.0-20200524003926-2c5affb30a03 h1:FjiybxAA4IL7VWbzhdskUDIXM2yIyE+/PRchayObTC4= -github.com/ianlancetaylor/demangle v0.0.0-20200524003926-2c5affb30a03/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/jessevdk/go-flags v1.4.0 h1:4IU2WS7AumrZ/40jfhf4QVDMsQwqA7VEHozFRrGARJA= -github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= -github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/ns3777k/go-shodan v1.0.2 h1:WScCf0IU3x9uQvV2YhQINzrnNBZ7KGMyhCjvrza2YWE= -github.com/ns3777k/go-shodan v3.1.0+incompatible h1:x+R8ZgUO6TKCVz9V5nz6ihSYF64BJWAaEPUylD6Zjy4= -github.com/ns3777k/go-shodan/v4 v4.2.0 h1:18R6axS4f+l37ic14BfjnmMo1dLgNTiPi6dtPXd9qwc= -github.com/ns3777k/go-shodan/v4 v4.2.0/go.mod h1:7kSWq/PQ/JCH6U4k2YjXRmnJKfPaJZAhOSMgAXRB23U= +github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= +github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= +github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= +github.com/go-ldap/ldap v3.0.2+incompatible/go.mod h1:qfd9rJvER9Q0/D/Sqn1DfHRoBp40uXYvFoEVrNEPqRc= +github.com/go-test/deep v1.0.2-0.20181118220953-042da051cf31/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= +github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= +github.com/hashicorp/go-hclog v0.0.0-20180709165350-ff2cf002a8dd/go.mod h1:9bjs9uLqI8l75knNv3lV1kA55veR+WUPSiKIWcQHudI= +github.com/hashicorp/go-hclog v0.8.0/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= +github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= +github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= +github.com/hashicorp/go-plugin v1.0.1/go.mod h1:++UyYGoz3o5w9ZzAdZxtQKrWWP+iqPBn3cQptSMzBuY= +github.com/hashicorp/go-retryablehttp v0.5.4/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs= +github.com/hashicorp/go-rootcerts v1.0.1/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8= +github.com/hashicorp/go-sockaddr v1.0.2/go.mod h1:rB4wwRAUzs07qva3c5SdrY/NEtAUjGlgmH/UkBUC97A= +github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-version v1.1.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= +github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/hashicorp/vault/api v1.0.4/go.mod h1:gDcqh3WGcR1cpF5AJz/B1UFheUEneMoIospckxBxk6Q= +github.com/hashicorp/vault/sdk v0.1.13/go.mod h1:B+hVj7TpuQY1Y/GPbCpffmgd+tSEwvhkWnjtSYCaS2M= +github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb/go.mod h1:+NfK9FKeTrX5uv1uIXGdwYDTeHna2qgaIlx54MXqjAM= +github.com/hashicorp/yamux v0.0.0-20181012175058-2f1d1f20f75d/go.mod h1:+NfK9FKeTrX5uv1uIXGdwYDTeHna2qgaIlx54MXqjAM= +github.com/jarcoal/httpmock v1.2.0 h1:gSvTxxFR/MEMfsGrvRbdfpRUMBStovlSRLw0Ep1bwwc= +github.com/jarcoal/httpmock v1.2.0/go.mod h1:oCoTsnAz4+UoOUIf5lJOWV2QQIW5UoeUI6aM2YnWAZk= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= +github.com/joho/godotenv v1.3.0 h1:Zjp+RcGpHhGlrMbJzXTrZZPrWj+1vfm90La1wgB6Bhc= +github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= +github.com/knadh/koanf v1.4.2 h1:2itp+cdC6miId4pO4Jw7c/3eiYD26Z/Sz3ATJMwHxIs= +github.com/knadh/koanf v1.4.2/go.mod h1:4NCo0q4pmU398vF9vq2jStF9MWQZ8JEDcDMHlDCr4h0= +github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= +github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/maxatome/go-testdeep v1.11.0 h1:Tgh5efyCYyJFGUYiT0qxBSIDeXw0F5zSoatlou685kk= +github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= +github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw= +github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw= +github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-testing-interface v0.0.0-20171004221916-a61a99592b77/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= +github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= +github.com/mitchellh/go-wordwrap v1.0.0/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUbP2oI0UX1GXzOo= +github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mitchellh/mapstructure v1.4.1 h1:CpVNEelQCZBooIPDn+AR3NpivK/TIKU8bDxdASFVQag= +github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/mitchellh/reflectwalk v1.0.0/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= +github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ= +github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= +github.com/npillmayer/nestext v0.1.3/go.mod h1:h2lrijH8jpicr25dFY+oAJLyzlya6jhnuG+zWp9L0Uk= +github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA= +github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= +github.com/pelletier/go-toml v1.7.0 h1:7utD74fnzVc/cpcyy8sjrlFr5vYpypUixARcHIMIGuI= +github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE= +github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= +github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/sirupsen/logrus v1.6.0 h1:UBcNElsrwanuuMsnGSlYmtmgbb23qDR5dG+6X6Oo89I= -github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= +github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= +github.com/rhnvrm/simples3 v0.6.1/go.mod h1:Y+3vYm2V7Y4VijFoJHHTrja6OgPrJ2cBti8dPGkC3sA= +github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= +github.com/ryanuber/go-glob v1.0.0/go.mod h1:807d1WSdnB0XRJzKNil9Om6lcp/3a0v4qIHxIXzX/Yc= +github.com/shadowscatcher/shodan v1.0.6 h1:T8G8DSK9NZmpOK68elQpspGWTwa2l5pGCFhIyGubZuA= +github.com/shadowscatcher/shodan v1.0.6/go.mod h1:8tnu38ME7RJgOfkJt/20JXK7bMvPTYCkLagO3IhI2SI= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= -github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -golang.org/x/sys v0.0.0-20190422165155-953cdadca894 h1:Cz4ceDQGXuKRnVBDTS23GTn/pU5OE2C0WrNTOYK1Uuc= -golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e h1:9vRrk9YW2BTzLP0VCB9ZDjU4cPqkg+IDWL7XgxA1yxQ= -golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200523222454-059865788121 h1:rITEj+UZHYC927n8GT97eC3zrpzXdb/voyeOuVKS46o= -golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= +go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/goleak v1.1.11 h1:wy28qYRKZgnJTxGxvye5/wgWr1EKjmUDGYox5mGlRlI= +go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= +go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4= +go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= +go.uber.org/zap v1.21.0 h1:WefMeulhovoZ2sYXz7st6K0sLj7bBhpiFaud4r4zST8= +go.uber.org/zap v1.21.0/go.mod h1:wjWOCqI0f2ZZrJF/UufIOkiC8ii6tm1iqIsLo76RfJw= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190129075346-302c3dd5f1cc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220615213510-4f61da869c0c h1:aFV+BgZ4svzjfabn8ERpuB4JI4N6/rdy1iusx77G3oU= +golang.org/x/sys v0.0.0-20220615213510-4f61da869c0c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.1-0.20181227161524-e6919f6577db/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190404172233-64821d5d2107/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.22.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d/go.mod h1:cuepJuh7vyXfUyUwEgHQXw849cJrilpS5NeIjOWESAw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/square/go-jose.v2 v2.3.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= +gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0 h1:hjy8E9ON/egN1tAYqKb61G10WtihqetD4sz2H+8nIeA= +gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/images/help1.png b/images/help1.png deleted file mode 100644 index 49f8818..0000000 Binary files a/images/help1.png and /dev/null differ diff --git a/log/log.go b/log/log.go new file mode 100644 index 0000000..5a51a8b --- /dev/null +++ b/log/log.go @@ -0,0 +1,115 @@ +package log + +import ( + "io" + "os" + + "go.uber.org/zap" + "go.uber.org/zap/zapcore" +) + +func (l *Logger) Debug(msg string, fields ...zap.Field) { + l.l.Debug(msg, fields...) +} + +func (l *Logger) Info(msg string, fields ...zap.Field) { + l.l.Info(msg, fields...) +} + +func (l *Logger) Warn(msg string, fields ...zap.Field) { + l.l.Warn(msg, fields...) +} + +func (l *Logger) Error(msg string, fields ...zap.Field) { + l.l.Error(msg, fields...) +} +func (l *Logger) DPanic(msg string, fields ...zap.Field) { + l.l.DPanic(msg, fields...) +} +func (l *Logger) Panic(msg string, fields ...zap.Field) { + l.l.Panic(msg, fields...) +} +func (l *Logger) Fatal(msg string, fields ...zap.Field) { + l.l.Fatal(msg, fields...) +} + +// function variables for all field types +// in github.com/uber-go/zap/field.go + +var ( + Skip = zap.Skip + Binary = zap.Binary + Bool = zap.Bool + Boolp = zap.Boolp + ByteString = zap.ByteString + Float64 = zap.Float64 + Float64p = zap.Float64p + Float32 = zap.Float32 + Float32p = zap.Float32p + Durationp = zap.Durationp + Any = zap.Any + Object = zap.Object + + Info = std.Info + Warn = std.Warn + Error = std.Error + DPanic = std.DPanic + Panic = std.Panic + Fatal = std.Fatal + Debug = std.Debug +) + +// not safe for concurrent use +func ResetDefault(l *Logger) { + std = l + Info = std.Info + Warn = std.Warn + Error = std.Error + DPanic = std.DPanic + Panic = std.Panic + Fatal = std.Fatal + Debug = std.Debug +} + +type Logger struct { + l *zap.Logger // zap ensure that zap.Logger is safe for concurrent use + level zapcore.Level +} + +var std, _ = New(os.Stdout, "INFO") + +func Default() *Logger { + return std +} + +func New(writer io.Writer, level string) (logger *Logger, err error) { + parsedAtomicLevel, err := zapcore.ParseLevel(level) + if err != nil { + return logger, err + } + if writer == nil { + return logger, err + } + cfg := zap.NewProductionConfig() + core := zapcore.NewCore( + zapcore.NewJSONEncoder(cfg.EncoderConfig), + zapcore.AddSync(writer), + zapcore.Level(parsedAtomicLevel), + ) + logger = &Logger{ + l: zap.New(core), + level: parsedAtomicLevel, + } + return logger, err +} + +func (l *Logger) Sync() error { + return l.l.Sync() +} + +func Sync() error { + if std != nil { + return std.Sync() + } + return nil +} diff --git a/main.go b/main.go new file mode 100644 index 0000000..e842673 --- /dev/null +++ b/main.go @@ -0,0 +1,24 @@ +package main + +import ( + "context" + "sync" + + "github.com/KaanSK/shomon/log" + "github.com/KaanSK/shomon/service" +) + +func main() { + var wg sync.WaitGroup + ctx := context.Background() + + srv, err := service.New(&wg, ctx) + if err != nil { + log.Error(err.Error()) + } + + wg.Add(1) + go srv.ListenStream() + wg.Wait() + +} diff --git a/pkg/conf/conf.go b/pkg/conf/conf.go deleted file mode 100644 index 6ece940..0000000 --- a/pkg/conf/conf.go +++ /dev/null @@ -1,13 +0,0 @@ -package conf - -// Options : argument struct -type Options struct { - Endpoint string `short:"e" long:"endpoint" description:"Full endpoint for alert API" default:"http://127.0.0.1:9000/api/alert"` - ShodanKey string `short:"s" long:"shodanKey" description:"Shodan Api Key" required:"true"` - CaseTemplate string `short:"c" long:"caseTemplate" description:"Case template for alert creation. Can be empty" default:""` - TheHiveKey string `short:"t" long:"theHiveKey" description:"TheHive api key" required:"true"` - Verbose bool `short:"v" long:"verbose" description:"Show verbose debug information"` -} - -// Config : Used to retrieve conf key/values. Globally available -var Config Options diff --git a/pkg/logwrapper/logwrapper.go b/pkg/logwrapper/logwrapper.go deleted file mode 100644 index f23bc6b..0000000 --- a/pkg/logwrapper/logwrapper.go +++ /dev/null @@ -1,25 +0,0 @@ -package logwrapper - -import ( - "os" - - "github.com/sirupsen/logrus" -) - -// StandardLogger enforces specific log message formats -type StandardLogger struct { - *logrus.Logger -} - -// Logger : Globally shared logging instance -var Logger = NewLogger() - -// NewLogger initializes the standard logger -func NewLogger() *StandardLogger { - var baseLogger = logrus.New() - var standardLogger = &StandardLogger{baseLogger} - standardLogger.SetLevel(logrus.InfoLevel) - - standardLogger.SetOutput(os.Stdout) - return standardLogger -} diff --git a/pkg/shodan/host.go b/pkg/shodan/host.go deleted file mode 100644 index 40ee363..0000000 --- a/pkg/shodan/host.go +++ /dev/null @@ -1,242 +0,0 @@ -// This package is derived from https://github.com/ns3777k/go-shodan - -//The MIT License (MIT) -// -//Copyright (c) 2015-present Safonov Nikita -// -//Permission is hereby granted, free of charge, to any person obtaining a copy -//of this software and associated documentation files (the "Software"), to deal -//in the Software without restriction, including without limitation the rights -//to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -//copies of the Software, and to permit persons to whom the Software is -//furnished to do so, subject to the following conditions: - -//The above copyright notice and this permission notice shall be included in all -//copies or substantial portions of the Software. - -//THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -//IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -//FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -//AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -//LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -//OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -//SOFTWARE. - -package shodan - -import ( - "encoding/json" - "errors" - "io/ioutil" - "math/big" - "net" - "net/http" - "strconv" - "strings" -) - -// Facet is a property to get summary information on. -type Facet struct { - Count int `json:"count"` - Value string `json:"value"` -} - -// IntString is string with custom unmarshaling. -type IntString string - -// UnmarshalJSON handles either a string or a number -// and casts it to string. -func (v *IntString) UnmarshalJSON(b []byte) error { - var s string - if err := json.Unmarshal(b, &s); err == nil { - *v = IntString(s) - return nil - } - - var n int - if err := json.Unmarshal(b, &n); err != nil { - return err - } - - *v = IntString(strconv.Itoa(n)) - - return nil -} - -// String method just returns string out of IntString. -func (v *IntString) String() string { - return string(*v) -} - -// GetErrorFromResponse used to getting errors from streaming api endpoint -func GetErrorFromResponse(r *http.Response) error { - errorResponse := new(struct { - Error string `json:"error"` - }) - message, err := ioutil.ReadAll(r.Body) - if err == nil { - if err := json.Unmarshal(message, errorResponse); err == nil { - return errors.New(errorResponse.Error) - } - - return errors.New(strings.TrimSpace(string(message))) - } - - return err -} - -// HostServicesOptions is options for querying services. -type HostServicesOptions struct { - History bool `url:"history,omitempty"` - Minify bool `url:"minify,omitempty"` -} - -// HostLocation is the location of the host. -type HostLocation struct { - City string `json:"city"` - RegionCode string `json:"region_code"` - AreaCode int `json:"area_code"` - Latitude float64 `json:"latitude"` - Longitude float64 `json:"longitude"` - Country string `json:"country_name"` - CountryCode string `json:"country_code"` - CountryCode3 string `json:"country_code3"` - Postal string `json:"postal_code"` - DMA int `json:"dma_code"` -} - -// HostDHParams is the Diffie-Hellman parameters if available. -type HostDHParams struct { - Prime string `json:"prime"` - PublicKey string `json:"public_key"` - Bits int `json:"bits"` - Generator *IntString `json:"generator"` - Fingerprint string `json:"fingerprint"` -} - -// HostTLSExtEntry contains id and name. -type HostTLSExtEntry struct { - ID int `json:"id"` - Name string `json:"name"` -} - -// HostCipher is a cipher description. -type HostCipher struct { - Version string `json:"version"` - Bits int `json:"bits"` - Name string `json:"name"` -} - -// HostCertificatePublicKey holds type and bits length of the key. -type HostCertificatePublicKey struct { - Type string `json:"type"` - Bits int `json:"bits"` -} - -// HostCertificateAttributes is an ordinary certificate attributes description. -type HostCertificateAttributes struct { - CountryName string `json:"C,omitempty"` - CommonName string `json:"CN,omitempty"` - Locality string `json:"L,omitempty"` - Organization string `json:"O,omitempty"` - StateOrProvinceName string `json:"ST,omitempty"` - OrganizationalUnit string `json:"OU,omitempty"` -} - -// HostCertificateExtension represent single cert extension. -type HostCertificateExtension struct { - Data string `json:"data"` - Name string `json:"name"` - IsCritical bool `json:"critical,omitempty"` -} - -// HostCertificate contains common certificate description. -type HostCertificate struct { - SignatureAlgorithm string `json:"sig_alg"` - IsExpired bool `json:"expired"` - Version int `json:"version"` - Serial *big.Int `json:"serial"` - Issued string `json:"issued"` - Expires string `json:"expires"` - Fingerprint map[string]string `json:"fingerprint"` - Issuer *HostCertificateAttributes `json:"issuer"` - Subject *HostCertificateAttributes `json:"subject"` - PublicKey *HostCertificatePublicKey `json:"pubkey"` - Extensions []*HostCertificateExtension `json:"extensions"` -} - -// HostSSL holds ssl host information. -type HostSSL struct { - Versions []string `json:"versions"` - Chain []string `json:"chain"` - DHParams *HostDHParams `json:"dhparams"` - TLSExt []*HostTLSExtEntry `json:"tlsext"` - Cipher *HostCipher `json:"cipher"` - Certificate *HostCertificate `json:"cert"` -} - -// HostData is all services that have been found on the given host IP. -type HostData struct { - Product string `json:"product"` - Hostnames []string `json:"hostnames"` - Version IntString `json:"version"` - Title string `json:"title"` - SSL *HostSSL `json:"ssl"` - IP net.IP `json:"ip_str"` - OS string `json:"os"` - Organization string `json:"org"` - ISP string `json:"isp"` - CPE []string `json:"cpe"` - Data string `json:"data"` - ASN string `json:"asn"` - Port int `json:"port"` - HTML string `json:"html"` - Banner string `json:"banner"` - Link string `json:"link"` - Transport string `json:"transport"` - Domains []string `json:"domains"` - Timestamp string `json:"timestamp"` - DeviceType string `json:"devicetype"` - Location *HostLocation `json:"location"` - ShodanData map[string]interface{} `json:"_shodan"` - Opts map[string]interface{} `json:"opts"` -} - -// Host is the all information about the host. -type Host struct { - OS string `json:"os"` - Ports []int `json:"ports"` - IP net.IP `json:"ip_str"` - ISP string `json:"isp"` - Hostnames []string `json:"hostnames"` - Organization string `json:"org"` - Vulnerabilities []string `json:"vulns"` - ASN string `json:"asn"` - LastUpdate string `json:"last_update"` - Data []*HostData `json:"data"` - HostLocation -} - -// HostQueryOptions is Shodan search query options. -type HostQueryOptions struct { - Query string `url:"query"` - Facets string `url:"facets,omitempty"` - Minify bool `url:"minify,omitempty"` - Page int `url:"page,omitempty"` -} - -// HostMatch is the search results with all matched hosts. -type HostMatch struct { - Total int `json:"total"` - Facets map[string][]*Facet `json:"facets"` - Matches []*HostData `json:"matches"` -} - -// HostQueryTokens is filters are being used by the query string and what -// parameters were provided to the filters. -type HostQueryTokens struct { - Filters []string `json:"filters"` - String string `json:"string"` - Errors []string `json:"errors"` - Attributes map[string]interface{} `json:"attributes"` -} diff --git a/pkg/shodan/shodan.go b/pkg/shodan/shodan.go deleted file mode 100644 index 8865d05..0000000 --- a/pkg/shodan/shodan.go +++ /dev/null @@ -1,113 +0,0 @@ -package shodan - -import ( - "bufio" - "bytes" - "crypto/md5" - "encoding/hex" - "encoding/json" - "errors" - "fmt" - "io" - "net/http" - "os" - - conf "github.com/KaanSK/shomon/pkg/conf" - lw "github.com/KaanSK/shomon/pkg/logwrapper" - "github.com/KaanSK/shomon/pkg/thehive" -) - -var ( - errShomonServiceStop = errors.New("listener service stopped") -) - -func parseResponse(destination interface{}, body io.Reader) error { - var err error - - if w, ok := destination.(io.Writer); ok { - _, err = io.Copy(w, body) - } else { - decoder := json.NewDecoder(body) - err = decoder.Decode(destination) - } - - return err -} - -func handleAlertStream(ch chan *HostData) { - defer func() { - close(ch) - }() - resp, err := http.Get("https://stream.shodan.io/shodan/alert?key=" + conf.Config.ShodanKey) - if err != nil { - lw.Logger.Error(err) - } - if resp.StatusCode != http.StatusOK { - err = GetErrorFromResponse(resp) - resp.Body.Close() - lw.Logger.Error(err) - if err.Error() == "No alerts specified" || err.Error() == "Invalid API key" { - os.Exit(1) - } - } - - reader := bufio.NewReader(resp.Body) - for { - banner := new(HostData) - chunk, err := reader.ReadBytes('\n') - if err != nil { - resp.Body.Close() - break - } - - chunk = bytes.TrimRight(chunk, "\n\r") - if len(chunk) == 0 { - continue - } - - if err := parseResponse(banner, bytes.NewBuffer(chunk)); err != nil { - resp.Body.Close() - lw.Logger.Error(err) - break - } - - ch <- banner - } -} - -// ListenAlerts : Used to listen streaming monitoring API -func ListenAlerts() error { - ch := make(chan *HostData) - go handleAlertStream(ch) - - lw.Logger.Info("listening process initiated") - - for { - banner, ok := <-ch - if !ok { - break - } - - hiveAlert := new(thehive.HiveAlert) - foundService := fmt.Sprintf("%s:%d", banner.IP, banner.Port) - hiveAlert.Title = fmt.Sprintf("Alert: %s", foundService) - hiveAlert.Description = "Test description" - - if conf.Config.CaseTemplate != "" { - hiveAlert.CaseTemplate = conf.Config.CaseTemplate - } - - hiveAlert.Source = "Shodan" - hash := md5.Sum([]byte(foundService)) - hiveAlert.SourceRef = hex.EncodeToString(hash[:]) - - lw.Logger.Info("triggered alarm for: " + hiveAlert.SourceRef) - - err := thehive.CreateAlert(hiveAlert) - if err != nil { - return err - } - lw.Logger.Info("created alert for " + hiveAlert.SourceRef) - } - return errShomonServiceStop -} diff --git a/pkg/thehive/alert.go b/pkg/thehive/alert.go deleted file mode 100644 index e0376b1..0000000 --- a/pkg/thehive/alert.go +++ /dev/null @@ -1,79 +0,0 @@ -package thehive - -import ( - "bytes" - "encoding/json" - "errors" - "io/ioutil" - "net/http" - "time" - - conf "github.com/KaanSK/shomon/pkg/conf" -) - -// HiveAlert : Alert structure -type HiveAlert struct { - CaseTemplate string `json:"caseTemplate,omitempty"` - Artifacts []interface{} `json:"artifacts"` - CreatedAt int64 `json:"createdAt"` - CreatedBy string `json:"createdBy"` - Date int64 `json:"date"` - Description string `json:"description"` - Follow bool `json:"follow"` - ID string `json:"id,omitempty"` - LastSyncDate int64 `json:"lastSyncDate"` - Severity int `json:"severity"` - Source string `json:"source"` - SourceRef string `json:"sourceRef"` - Status string `json:"status"` - Title string `json:"title"` - Tlp int `json:"tlp"` - Type string `json:"type"` - User string `json:"user,omitempty"` -} - -func getResponseJSON(url string, target interface{}, input interface{}) (interface{}, error) { - var netClient = &http.Client{ - Timeout: time.Second * 10, - } - payload, err := json.Marshal(input) - - req, err := http.NewRequest("POST", url, bytes.NewBuffer(payload)) - if err != nil { - return nil, err - } - req.Header.Set("Authorization", "Bearer "+conf.Config.TheHiveKey) - req.Header.Set("Content-Type", "application/json") - - resp, err := netClient.Do(req) - if err != nil { - return nil, err - } - defer resp.Body.Close() - - if resp.StatusCode != 201 { - bodyBytes, err := ioutil.ReadAll(resp.Body) - if err != nil { - return nil, err - } - bodyString := string(bodyBytes) - return nil, errors.New(bodyString) - } - - err = json.NewDecoder(resp.Body).Decode(&target) - if err != nil { - return nil, err - } - return target, nil - -} - -// CreateAlert : Used to create alert on thehive -func CreateAlert(alertObject *HiveAlert) error { - endpoint := conf.Config.Endpoint - _, err := getResponseJSON(endpoint, HiveAlert{}, *alertObject) - if err != nil { - return err - } - return nil -} diff --git a/producer/producer.go b/producer/producer.go new file mode 100644 index 0000000..4fb9531 --- /dev/null +++ b/producer/producer.go @@ -0,0 +1,11 @@ +package producer + +import ( + "context" + + "github.com/shadowscatcher/shodan/models" +) + +type Producer interface { + ListenAlerts(ctx context.Context) (chan models.Service, error) +} diff --git a/producer/shodan_stream.go b/producer/shodan_stream.go new file mode 100644 index 0000000..705e3d4 --- /dev/null +++ b/producer/shodan_stream.go @@ -0,0 +1,41 @@ +package producer + +import ( + "context" + "errors" + "net/http" + + "github.com/shadowscatcher/shodan" + "github.com/shadowscatcher/shodan/models" +) + +type ShodanStream struct { + client *shodan.StreamClient +} + +func GetShodanStreamClient(ShodanKey string, httpClient *http.Client) (*ShodanStream, error) { + if ShodanKey == "" { + return nil, errors.New("empty Shodan API key") + } + + if httpClient == nil { + return nil, errors.New("HTTP client is nil") + } + + client, err := shodan.GetStreamClient(ShodanKey, httpClient) + if err != nil { + return nil, err + } + + return &ShodanStream{ + client: client, + }, nil +} + +func (ss *ShodanStream) ListenAlerts(ctx context.Context) (chan models.Service, error) { + alertChan, err := ss.client.Alerts(ctx) + if err != nil { + return nil, err + } + return alertChan, nil +} diff --git a/producer/shodan_webhook.go b/producer/shodan_webhook.go new file mode 100644 index 0000000..0bde2ef --- /dev/null +++ b/producer/shodan_webhook.go @@ -0,0 +1,76 @@ +package producer + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + + "github.com/shadowscatcher/shodan/models" +) + +type ShodanWebhook struct { + ShodanKey string + Endpoint string + Port int +} + +func GetShodanWebhook(shodanKey string, endpoint string, port int) (*ShodanWebhook, error) { + if shodanKey == "" { + return nil, errors.New("empty API key") + } + + if endpoint == "" { + return nil, errors.New("endpoint is empty") + } + + return &ShodanWebhook{ + ShodanKey: shodanKey, + Endpoint: endpoint, + Port: port, + }, nil +} + +func (sw *ShodanWebhook) ListenAlerts(ctx context.Context) (chan models.Service, error) { + bannerChan := make(chan models.Service) + + bannerHandler := banner(bannerChan) + http.HandleFunc(sw.Endpoint, bannerHandler) + go func() { + defer close(bannerChan) + http.ListenAndServe(fmt.Sprintf(":%d", sw.Port), nil) + }() + + return bannerChan, nil + +} + +func banner(bannerChan chan models.Service) func(http.ResponseWriter, *http.Request) { + return func(w http.ResponseWriter, req *http.Request) { + banner := models.Service{} + /* alertID := req.Header.Get("SHODAN-ALERT-ID") + alertName := req.Header.Get("SHODAN-ALERT-NAME") + alertTrigger := req.Header.Get("SHODAN-ALERT-TRIGGER") + alertVerify := req.Header.Get("SHODAN-SIGNATURE-SHA1") + if containsEmpty(alertID, alertName, alertTrigger, alertVerify) { + w.WriteHeader(http.StatusBadRequest) + return + } */ + if err := json.NewDecoder(req.Body).Decode(&banner); err != nil { + w.WriteHeader(http.StatusBadRequest) + } else { + bannerChan <- banner + w.WriteHeader(http.StatusOK) + } + } +} + +func containsEmpty(ss ...string) bool { + for _, s := range ss { + if s == "" { + return true + } + } + return false +} diff --git a/service/service.go b/service/service.go new file mode 100644 index 0000000..a8452fe --- /dev/null +++ b/service/service.go @@ -0,0 +1,124 @@ +package service + +import ( + "context" + "crypto/md5" + "encoding/hex" + "encoding/json" + "fmt" + "net/http" + "os" + "sync" + + "github.com/KaanSK/shomon/conf" + "github.com/KaanSK/shomon/log" + "github.com/KaanSK/shomon/producer" + "github.com/KaanSK/shomon/thehive" + "github.com/shadowscatcher/shodan/models" +) + +type Result struct { + Message string + Error error +} + +type Service struct { + Config conf.ShomonConfig + HiveClient *thehive.TheHiveClient + ShodanClient producer.Producer + Logger log.Logger + wg *sync.WaitGroup + ctx context.Context +} + +func New(wg *sync.WaitGroup, ctx context.Context) (srv Service, err error) { + shomonConf, err := conf.New() + if err != nil { + log.Fatal(err.Error()) + } + logger, err := log.New(os.Stdout, shomonConf.LogLevel) + if err != nil { + log.Fatal(err.Error()) + } + + hiveClient, err := thehive.GetHiveClient(shomonConf.HiveUrl, shomonConf.HiveKey, http.DefaultClient) + if err != nil { + log.Fatal(err.Error()) + } + var bannerClient producer.Producer + if shomonConf.Webhook { + bannerClient, err = producer.GetShodanWebhook(shomonConf.ShodanKey, shomonConf.WebhookEndpoint, shomonConf.WebhookPort) + log.Info("webhook mode is activated") + } else { + bannerClient, err = producer.GetShodanStreamClient(shomonConf.ShodanKey, http.DefaultClient) + log.Info("stream listening mode is activated") + } + + if err != nil { + log.Fatal(err.Error()) + } + + logger.Debug(fmt.Sprintf("Config= %s", shomonConf.Print())) + + srv.Config = shomonConf + srv.Logger = *logger + srv.wg = wg + srv.ctx = ctx + srv.HiveClient = hiveClient + srv.ShodanClient = bannerClient + + return srv, nil +} + +func (s *Service) ListenStream() error { + defer s.wg.Done() + + s.Logger.Info("starting service...") + + alertChan, err := s.ShodanClient.ListenAlerts(s.ctx) + if err != nil { + return err + } + + for banner := range alertChan { + s.wg.Add(1) + go func() { + id, err := s.ProcessAlert(banner) + if err != nil { + s.Logger.Error(err.Error()) + return + } + s.Logger.Info(fmt.Sprintf("Alert %s Created for %s", id, banner.IPstr)) + }() + } + + return nil +} + +func (s *Service) ProcessAlert(banner models.Service) (id string, err error) { + defer s.wg.Done() + s.Logger.Debug(PrintBanner(banner)) + + foundService := fmt.Sprintf("%s:%d", banner.IPstr, banner.Port) + foundServiceHash := md5.Sum([]byte(foundService)) + + alert := thehive.NewAlert() + alert.Type = s.Config.HiveType + alert.Source = "Shodan" + alert.SourceRef = hex.EncodeToString(foundServiceHash[:6]) + alert.Tags = s.Config.HiveTags + alert.Title = fmt.Sprintf("Shodan Alert: %s", foundService) + alert.AddObservable("ip", banner.IPstr) + alert.ExternalLink = fmt.Sprintf("https://www.shodan.io/host/%s", banner.IPstr) + alert.Description = fmt.Sprintf("[Alert Link](%s)", alert.ExternalLink) + if s.Config.IncludeBanner { + alert.Description = fmt.Sprintf("%s\n\n```\n\n%s\n\n```", alert.Description, PrintBanner(banner)) + } + id, err = s.HiveClient.CreateAlert(alert) + return id, err +} + +func PrintBanner(banner models.Service) string { + s, _ := json.MarshalIndent(banner, "", "\t") + return string(s) +} diff --git a/service/service_stream_test.go b/service/service_stream_test.go new file mode 100644 index 0000000..372b290 --- /dev/null +++ b/service/service_stream_test.go @@ -0,0 +1,145 @@ +package service + +import ( + "context" + "encoding/binary" + "encoding/json" + "fmt" + "math/rand" + "net" + "net/http" + "os" + "sync" + "testing" + "time" + + "github.com/KaanSK/shomon/log" + "github.com/KaanSK/shomon/thehive" + "github.com/jarcoal/httpmock" + "github.com/shadowscatcher/shodan/models" +) + +type MockStream struct { +} + +func (ss *MockStream) ListenAlerts(ctx context.Context) (chan models.Service, error) { + bannerChan := make(chan models.Service) + bannerJson := `{ + "hash": 1015805840, + "timestamp": "2021-01-28T04:16:08.387364", + "hostnames": [ + "177-70-193-184-msltr-cw-1.visaonet.com.br" + ], + "org": "TESTORG", + "data": "SIP/2.0 404 Not Found\r\nFrom: ;tag=root\r\nTo: ;tag=b235f0-b146c1b8-13c4-50029-ec2e4-6c44dfcf-ec2e4\r\nCall-ID: 50000\r\nCSeq: 42 OPTIONS\r\nVia: SIP/2.0/UDP nm;received=224.238.62.40;rport=26810;branch=foo\r\nSupported: replaces,100rel,timer\r\nAccept: application/sdp\r\nAllow: INVITE,ACK,CANCEL,BYE,OPTIONS,REFER,INFO,NOTIFY,PRACK,MESSAGE\r\nContent-Length: 0\r\n\r\n", + "port": 5060, + "transport": "udp", + "info": "SIP end point; Status: 404 Not Found", + "isp": "L M Tiko Kamide - Sva", + "asn": "AS28359", + "location": { + "country_code3": null, + "city": "Jardim Alegre", + "region_code": "PR", + "postal_code": null, + "longitude": -51.7213, + "country_code": "BR", + "latitude": -24.2123, + "country_name": "Brazil", + "area_code": null, + "dma_code": null + }, + "ip": 2974204344, + "domains": [ + "visaonet.com.br" + ], + "ip_str": "177.70.193.184", + "_id": "45ad6383-1b1d-4c5d-8584-d586fbdefbc3", + "os": null, + "_shodan": { + "crawler": "bf213bc419cc8491376c12af31e32623c1b6f467", + "options": {}, + "id": "220ef463-756f-4446-a89f-685053da8865", + "module": "sip", + "ptr": true + }, + "opts": {} + }` + banner := models.Service{} + buf := make([]byte, 4) + err := json.Unmarshal([]byte(bannerJson), &banner) + if err != nil { + return nil, err + } + rand.Seed(time.Now().UTC().UnixNano()) + go func() { + for i := 0; i < 5; i++ { + time.Sleep(1 * time.Second) + ip := rand.Uint32() + binary.LittleEndian.PutUint32(buf, ip) + banner.IPstr = fmt.Sprintf("%s", net.IP(buf)) + bannerChan <- banner + } + close(bannerChan) + }() + + return bannerChan, nil +} + +func TestListenStreamAlerts(t *testing.T) { + ms := &MockStream{} + var wg sync.WaitGroup + ctx := context.Background() + logger, err := log.New(os.Stdout, "ERROR") + if err != nil { + log.Fatal(err.Error()) + } + + httpmock.Activate() + defer httpmock.DeactivateAndReset() + endpoint := "https://test.local" + + httpmock.RegisterResponder("POST", endpoint+"/api/v1/alert", + func(req *http.Request) (*http.Response, error) { + alert := thehive.Alert{} + if err := json.NewDecoder(req.Body).Decode(&alert); err != nil { + return httpmock.NewStringResponse(400, ""), nil + } + alert.Id = fmt.Sprintf("~%d", rand.Intn(1000000)) + resp, err := httpmock.NewJsonResponse(201, alert) + if err != nil { + return httpmock.NewStringResponse(500, ""), nil + } + return resp, nil + }, + ) + + hiveClient, err := thehive.GetHiveClient(endpoint, "TEST", http.DefaultClient) + if err != nil { + t.Errorf(err.Error()) + } + + srv := Service{ + ShodanClient: ms, + HiveClient: hiveClient, + wg: &wg, + ctx: ctx, + Logger: *logger, + } + + alertChan, err := srv.ShodanClient.ListenAlerts(ctx) + if err != nil { + t.Errorf(err.Error()) + } + + for banner := range alertChan { + srv.wg.Add(1) + go func() { + _, err := srv.ProcessAlert(banner) + if err != nil { + t.Errorf(err.Error()) + } + }() + } + wg.Wait() +} diff --git a/shomon.go b/shomon.go deleted file mode 100644 index d2f00ab..0000000 --- a/shomon.go +++ /dev/null @@ -1,44 +0,0 @@ -package main - -import ( - "os" - - "github.com/KaanSK/shomon/pkg/conf" - lw "github.com/KaanSK/shomon/pkg/logwrapper" - "github.com/KaanSK/shomon/pkg/shodan" - "github.com/jessevdk/go-flags" - "github.com/sirupsen/logrus" -) - -func init() { - parser := flags.NewParser(&conf.Config, flags.Default) - _, err := parser.Parse() - if err != nil { - os.Exit(1) - } - if conf.Config.Verbose { - lw.Logger.Formatter = &logrus.JSONFormatter{} - lw.Logger.SetReportCaller(true) - lw.Logger.SetLevel(logrus.DebugLevel) - } -} - -func neverExit() { - defer func() { - if err := recover(); err != nil { - lw.Logger.Error(err) - go neverExit() - } - }() - err := shodan.ListenAlerts() - if err != nil { - lw.Logger.Error(err) - go neverExit() - } -} - -func main() { - lw.Logger.Info("main process started") - go neverExit() - select {} -} diff --git a/thehive/models.go b/thehive/models.go new file mode 100644 index 0000000..c3ff177 --- /dev/null +++ b/thehive/models.go @@ -0,0 +1,18 @@ +package thehive + +type Alert struct { + Id string `json:"_id,omitempty"` + Type string `json:"type"` + Source string `json:"source"` + SourceRef string `json:"sourceRef"` + Title string `json:"title"` + Description string `json:"description"` + ExternalLink string `json:"externalLink"` + Tags []string `json:"tags"` + Observables []Observable `json:"observables"` +} + +type Observable struct { + DataType string `json:"dataType"` + Data string `json:"data"` +} diff --git a/thehive/thehive.go b/thehive/thehive.go new file mode 100644 index 0000000..af65dc1 --- /dev/null +++ b/thehive/thehive.go @@ -0,0 +1,114 @@ +package thehive + +import ( + "bytes" + "encoding/json" + "errors" + "io/ioutil" + "log" + "net/http" +) + +type TheHiveClient struct { + apiKey string + url string + HTTP *http.Client + Logger log.Logger +} + +func GetHiveClient(url string, key string, client *http.Client) (*TheHiveClient, error) { + /* if key == "" { + return nil, errors.New("empty Hive API key") + } + + if client == nil { + return nil, errors.New("HTTP client is nil") + } */ + + return &TheHiveClient{ + apiKey: key, + url: url, + HTTP: client, + }, nil +} + +func NewAlert() Alert { + return Alert{} +} + +func (a *Alert) AddObservable(obsType string, obs string) { + obsInstance := Observable{ + Data: obs, + DataType: obsType, + } + a.Observables = append(a.Observables, obsInstance) +} + +func (s *TheHiveClient) CreateAlert(alert Alert) (id string, err error) { + if s == nil { + return id, errors.New("not initialized hive client") + } + payload, err := json.Marshal(alert) + if err != nil { + return id, err + } + req, err := http.NewRequest("POST", s.url+"/api/v1/alert", bytes.NewBuffer(payload)) + if err != nil { + return id, err + } + req.Header.Set("Authorization", "Bearer "+s.apiKey) + req.Header.Set("Content-Type", "application/json") + + resp, err := s.HTTP.Do(req) + if err != nil { + return id, err + } + defer resp.Body.Close() + body, err := ioutil.ReadAll(resp.Body) + if resp.StatusCode != http.StatusCreated { + if err != nil { + return id, err + } + return id, errors.New(string(body)) + } + createdAlert := NewAlert() + err = json.Unmarshal(body, &createdAlert) + if err != nil { + return id, errors.New(err.Error()) + } + return createdAlert.Id, nil +} + +type DeleteAlertsInput struct { + Ids []string `json:"ids"` +} + +func (s *TheHiveClient) DeleteAlerts(ids []string) error { + if s == nil { + return errors.New("not initialized hive client") + } + payload, err := json.Marshal(&DeleteAlertsInput{Ids: ids}) + if err != nil { + return err + } + req, err := http.NewRequest("POST", s.url+"/api/v1/alert/delete/_bulk", bytes.NewBuffer(payload)) + if err != nil { + return err + } + req.Header.Set("Authorization", "Bearer "+s.apiKey) + req.Header.Set("Content-Type", "application/json") + + resp, err := s.HTTP.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusNoContent { + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return err + } + return errors.New(string(body)) + } + return nil +} diff --git a/thehive/thehive_test.go b/thehive/thehive_test.go new file mode 100644 index 0000000..4f595f9 --- /dev/null +++ b/thehive/thehive_test.go @@ -0,0 +1,61 @@ +package thehive + +import ( + "encoding/json" + "math/rand" + "net/http" + "strconv" + "testing" + + "github.com/jarcoal/httpmock" +) + +func TestGetHiveClient(t *testing.T) { + apiKey := "TeST" + endpoint := "https://test.local" + client, err := GetHiveClient(endpoint, apiKey, http.DefaultClient) + if err != nil { + t.Errorf(err.Error()) + } + if client.apiKey != apiKey || client.url != endpoint { + t.Errorf("Client could not be initalized with proper input") + } +} + +func TestCreateAlert(t *testing.T) { + httpmock.Activate() + defer httpmock.DeactivateAndReset() + endpoint := "https://test.local" + + httpmock.RegisterResponder("POST", endpoint+"/api/v1/alert", + func(req *http.Request) (*http.Response, error) { + alert := Alert{} + if err := json.NewDecoder(req.Body).Decode(&alert); err != nil { + return httpmock.NewStringResponse(400, ""), nil + } + resp, err := httpmock.NewJsonResponse(201, alert) + if err != nil { + return httpmock.NewStringResponse(500, ""), nil + } + return resp, nil + }, + ) + + hiveClient, err := GetHiveClient(endpoint, "TEST", http.DefaultClient) + if err != nil { + t.Errorf(err.Error()) + } + alert := NewAlert() + alert.Description = "TestDescription" + alert.Type = "TEST_TYPE" + alert.Title = "TestTitle" + alert.Tags = []string{"test1", "test2"} + alert.Source = "Shodan" + alert.SourceRef = strconv.Itoa(100000 + rand.Intn(900000)) + alert.AddObservable("ip", "1.1.1.1") + _, err = hiveClient.CreateAlert(alert) + if err != nil { + t.Errorf(err.Error()) + } + +}