Issue #192 - Salvar Sitemap no CDN do Frontend (#318)

* poder usar "s3" localmente atraves do minio
* salvar sitemap no s3 do frontend
* updates sitemap on deploy
* hosts on readme.md

Author: lucassabreu

Makefile

@@ -55,8 +55,10 @@ migrations-on-docker: ## run migrations inside docker
 	docker-compose up -d
 	docker-compose exec golang sql-migrate up -config=devops/dbconfig.yml -env=docker
 
-serve: ## start server
-	docker-compose up -d postgres
+start-dependences: ## docker up all container dependencies
+	docker-compose up -d postgres minio images-server
+
+serve: start-dependences ## start server
 	cd server && go run main.go serve
 
 install-on-docker: ## install dependences from docker
@@ -67,8 +69,7 @@ serve-on-docker: ## start the server inside docker
 	docker-compose up -d
 	docker-compose exec golang sh -c "cd server && go run main.go serve"
 
-serve-watch: ## start server with hot reload (bin=vitrine-social)
-	docker-compose up -d postgres
+serve-watch: start-dependences ## start server with hot reload (bin=vitrine-social)
 	go get -u github.com/codegangsta/gin
 	cd server; API_PORT=8001 gin --port 8000 --appPort 8001 --bin $(bin) run serve
 
@@ -87,9 +88,12 @@ docs-serve: ## start a server with the docs
 docs-build: ## build the docs
 	cd docs && make build
 
-docs-open:
+docs-open: ## opens the docs on your browser
 	$$BROWSER docs/index.html
 
+open-minio: ## opens the docs on your browser
+	$$BROWSER localhost:9000
+
 # Absolutely awesome: http://marmelab.com/blog/2016/02/29/auto-documented-makefile.html
 help: ## show this help
 	@grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'

README.md

@@ -6,7 +6,6 @@ Vitrine Social [![Codacy Badge](https://api.codacy.com/project/badge/Grade/5d73b
 
 O controle das tarefas e do progresso das mesmas estão sendo feitas no Waffle. Clique aqui para acompanhar: https://waffle.io/Coderockr/vitrine-social
 
-
 ## Instalação Backend (Go)
 
 Estamos utilizando [Go Modules](https://github.com/golang/go/wiki/Modules) nesse projeto, por isso a pasta do projeto precisa ficar fora do seu `GOPATH`, ou terá que adicionar a ENV `GO111MODULE` como `on` em seu ambiente para que o projeto funcione dentro do `GOPATH`.
@@ -25,6 +24,17 @@ make migrations # isso pode falhar por causa do warmup do postgres
 make serve # agora esta rodando :)
 ```
 
+### Domínios e Subdomínios locais
+
+Incluir os seguintes domínios no seu `/etc/hosts` deve agilizar o setup do seu projeto:
+
+```sh
+127.0.0.1 api.vitrinesocial.test # usar porta 8000 (golang)
+127.0.0.1 images.vitrinesocial.test # usar porta 7000 (images-server)
+127.0.0.1 minio.vitrinesocial.test # usar porta 9000 (minio)
+127.0.0.1 vitrinesocial.test # usar porta 3000 (frontend)
+```
+
 ## Instalação Frontend (React)
 
 ```sh

devops/deploy.sh

@@ -15,3 +15,6 @@ ssh -i ./devops/vitrinesocial.pem -t $DEPLOY_USER@$DEPLOY_HOST 'sudo systemctl s
 
 # Run Migrations
 sql-migrate up -config=devops/dbconfig.yml -env=production
+
+# Update sitemap
+ssh -i ./devops/vitrinesocial.pem -t $DEPLOY_USER@$DEPLOY_HOST './vitrine-social/vitrine-social sitemap-generate'

docker-compose.yml

@@ -28,3 +28,23 @@ services:
     working_dir: /vitrine-social
     links:
       - postgres
+  minio:
+    image: minio/minio:RELEASE.2018-10-18T00-28-58Z
+    environment:
+      MINIO_ACCESS_KEY: configAccessKeyId
+      MINIO_SECRET_KEY: configSecretKey
+    volumes:
+      - "./.data/minio:/data"
+      - "./.data/minio-config:/root/.minio"
+    ports:
+      - "9000:9000"
+    entrypoint: sh
+    command: -c 'mkdir -p /data/images.vitrinesocial.test /data/vitrinesocial.test && /usr/bin/minio server /data'
+  images-server:
+    image: halverneus/static-file-server:latest
+    volumes:
+      - "./.data/minio/images.vitrinesocial.test:/web"
+    ports:
+      - 7000:8080
+    depends_on:
+      - minio

server/cmd/sitemap.go

@@ -15,13 +15,16 @@
 package cmd
 
 import (
+	"bytes"
 	"fmt"
 	"log"
 	"os"
 
 	"github.com/Coderockr/vitrine-social/server/db"
 	"github.com/Coderockr/vitrine-social/server/db/repo"
 	"github.com/Coderockr/vitrine-social/server/model"
+	"github.com/Coderockr/vitrine-social/server/storage"
+	"github.com/graymeta/stow"
 	"github.com/spf13/cobra"
 	"github.com/thecodenation/go-sitemap-generator/stm"
 )
@@ -42,6 +45,12 @@ func sitemapFunc(cmd *cobra.Command, args []string) {
 	var o []*model.Organization
 	sm := generateSitemap()
 
+	storageContainer, err := storage.ConnectFrontend()
+	if err != nil {
+		log.Fatal(err.Error())
+		os.Exit(1)
+	}
+
 	conn, err := db.GetFromEnv()
 	if err != nil {
 		log.Fatal(err.Error())
@@ -66,7 +75,7 @@ func sitemapFunc(cmd *cobra.Command, args []string) {
 			sm.Add(stm.URL{"loc": fmt.Sprintf("%s/v1/need/%d/share", os.Getenv("API_URL"), k.ID), "changefreq": "hourly", "priority": 0.8})
 		}
 	}
-	err = saveSitemap(sm)
+	err = saveSitemap(sm, storageContainer)
 	if err != nil {
 		log.Fatal(err.Error())
 		os.Exit(1)
@@ -83,17 +92,16 @@ func generateSitemap() *stm.Sitemap {
 	return sm
 }
 
-func saveSitemap(sm *stm.Sitemap) error {
-	f, err := os.Create("../frontend/public/sitemap.xml")
-	if err != nil {
-		return err
-	}
-	defer f.Close()
-	_, err = f.Write(sm.XMLContent())
-	if err != nil {
-		return err
-	}
-	err = f.Sync()
+func saveSitemap(sm *stm.Sitemap, c stow.Container) error {
+	content := sm.XMLContent()
+
+	_, err := c.Put(
+		"sitemap.xml",
+		bytes.NewBuffer(content),
+		int64(len(content)),
+		nil,
+	)
+
 	if err != nil {
 		return err
 	}

server/config/dev.env.dist

@@ -3,21 +3,27 @@ DATABASE_USER=postgres
 DATABASE_PASSWORD=postgres
 DATABASE_HOST=127.0.0.1
 DATABASE_PORT=5432
-API_URL=http://vitrinesocial.test
+API_URL=http://api.vitrinesocial.test
 API_PORT=8000
 FRONTEND_URL=https://vitrinesocial.org
 VITRINESOCIAL_PRIVATE_KEY=ThisIsASecretSalt
 VITRINESOCIAL_PUBLIC_KEY=
 VITRINESOCIAL_SIGNING_METHOD=HS256
+
 STORAGE=s3
-IMAGE_STORAGE_BASE_URL=http://images.vitrinesocial.org/
-STORAGE_S3_BUCKET_NAME=images.vitrinesocial.org
+IMAGE_STORAGE_BASE_URL=http://images.vitrinesocial.test:7000/
+STORAGE_S3_BUCKET_NAME=images.vitrinesocial.test
+STORAGE_S3_ENDPOINT=http://127.0.0.1:9000
 STORAGE_S3_CONFIG_ACCESS_KEY_ID=configAccessKeyId
 STORAGE_S3_CONFIG_SECRET_KEY=configSecretKey
 STORAGE_S3_CONFIG_REGION=configRegion
 STORAGE_LOCAL_PATH=
 STORAGE_GOOGLE_PATH=
 
+STORAGE_LOCAL_PATH_FRONTEND=../frontend/public
+STORAGE_GOOGLE_PATH_FRONTEND=
+STORAGE_S3_BUCKET_NAME_FRONTEND=vitrinesocial.test
+
 BUGSNAG_KEY=
 
 MAIL_METHOD=

server/storage/storage.go

@@ -4,26 +4,36 @@ import (
 	"os"
 
 	"github.com/graymeta/stow"
-	// support Google storage
-	google "github.com/graymeta/stow/google"
-	// support local storage
-	local "github.com/graymeta/stow/local"
-	// support s3 storage
-	s3 "github.com/lucassabreu/stow/s3"
+	google "github.com/graymeta/stow/google" // support Google storage
+	local "github.com/graymeta/stow/local"   // support local storage
+	s3 "github.com/lucassabreu/stow/s3"      // support s3 storage
 )
 
 // Connect to storage and return the container
 func Connect() (stow.Container, error) {
+	return connectTo("")
+}
+
+// ConnectFrontend will connect to the frontends file container
+func ConnectFrontend() (stow.Container, error) {
+	return connectTo("FRONTEND")
+}
+
+func connectTo(sufix string) (stow.Container, error) {
 	var kind string
 	var config stow.ConfigMap
 	var containerName string
 
+	if len(sufix) > 0 {
+		sufix = "_" + sufix
+	}
+
 	if os.Getenv("STORAGE") == "local" {
 		kind = "local"
 		config = stow.ConfigMap{
 			local.ConfigKeyPath: os.Getenv("STORAGE_LOCAL_PATH"),
 		}
-		containerName = os.Getenv("STORAGE_LOCAL_PATH")
+		containerName = os.Getenv("STORAGE_LOCAL_PATH" + sufix)
 	}
 
 	if os.Getenv("STORAGE") == "s3" {
@@ -33,7 +43,12 @@ func Connect() (stow.Container, error) {
 			s3.ConfigSecretKey:   os.Getenv("STORAGE_S3_CONFIG_SECRET_KEY"),
 			s3.ConfigRegion:      os.Getenv("STORAGE_S3_CONFIG_REGION"),
 		}
-		containerName = os.Getenv("STORAGE_S3_BUCKET_NAME")
+
+		if endpoint := os.Getenv("STORAGE_S3_ENDPOINT"); len(endpoint) > 0 {
+			config[s3.ConfigEndpoint] = endpoint
+		}
+
+		containerName = os.Getenv("STORAGE_S3_BUCKET_NAME" + sufix)
 	}
 
 	if os.Getenv("STORAGE") == "google" {