Merge pull request #8 from lovoo/bbq-fix

bbq: ignore unexported fields, updated goka

bbq/infer_schema.go

@@ -20,6 +20,7 @@ import (
 	"fmt"
 	"log"
 	"reflect"
+	"strings"
 	"time"
 
 	"cloud.google.com/go/bigquery"
@@ -103,7 +104,7 @@ func inferFieldSchema(rt reflect.Type) (*bigquery.FieldSchema, error) {
 	case reflect.Interface:
 		return &bigquery.FieldSchema{Required: true, Type: bigquery.StringFieldType}, nil
 	default:
-		log.Printf("unsupported type is: %+v", rt.Kind())
+		log.Printf("unsupported type for field %s is: %+v", rt.Name(), rt.Kind())
 		return nil, errUnsupportedFieldType
 	}
 }
@@ -122,6 +123,15 @@ func inferFields(rt reflect.Type) (bigquery.Schema, error) {
 		if field.Tag.Get("json") == "-" {
 			continue
 		}
+		if field.Name == "" {
+			continue
+		}
+
+		// if the field is not exported, drop it
+		if strings.ToUpper(field.Name[:1]) != field.Name[:1] {
+			continue
+		}
+
 		inferredSchema, err := inferFieldSchema(field.Type)
 		if err != nil {
 			return nil, err

bbq/infer_schema_test.go

@@ -0,0 +1,25 @@
+package bbq
+
+import "testing"
+
+type sometype struct {
+	unexported int
+	Exported   int
+}
+
+func TestInferSchema(t *testing.T) {
+
+	t.Run("ignore-unexported", func(t *testing.T) {
+		schema, err := inferSchema(new(sometype))
+		if err != nil {
+			t.Fatalf("Error infering schema. %v", err)
+		}
+
+		if len(schema) != 1 {
+			t.Fatalf("schema should contain only the exported field")
+		}
+		if schema[0].Name != "Exported" {
+			t.Fatalf("the wrong field was exported from the struct")
+		}
+	})
+}

examples/bbq/main.go

@@ -33,7 +33,7 @@ func main() {
 		},
 	}
 
-	bbq, err := bbq.NewBbq("gcp-project", "target-dataset", tables)
+	bbq, err := bbq.NewBbq("gcp-project", "target-dataset", tables, "metrics")
 	if err != nil {
 		log.Fatalf("Unable to create new BBQ: %v", err)
 	}

go.mod

@@ -0,0 +1,30 @@
+module github.com/lovoo/goka-tools
+
+go 1.13
+
+require (
+	cloud.google.com/go v0.56.0
+	cloud.google.com/go/bigquery v1.5.0
+	cloud.google.com/go/pubsub v1.3.1 // indirect
+	github.com/Shopify/sarama v1.26.1
+	github.com/facebookgo/ensure v0.0.0-20200202191622-63f1cf65ac4c
+	github.com/facebookgo/stack v0.0.0-20160209184415-751773369052 // indirect
+	github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4 // indirect
+	github.com/klauspost/compress v1.10.4 // indirect
+	github.com/kr/text v0.2.0 // indirect
+	github.com/lovoo/goka v0.9.0-beta3
+	github.com/onsi/ginkgo v1.10.1 // indirect
+	github.com/onsi/gomega v1.7.0 // indirect
+	github.com/pierrec/lz4 v2.5.1+incompatible // indirect
+	github.com/prometheus/client_golang v1.5.1
+	github.com/prometheus/procfs v0.0.11 // indirect
+	github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0 // indirect
+	github.com/stretchr/testify v1.5.1 // indirect
+	golang.org/x/crypto v0.0.0-20200420201142-3c4aac89819a // indirect
+	golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e
+	golang.org/x/sys v0.0.0-20200408040146-ea54a3c99b9b // indirect
+	golang.org/x/tools v0.0.0-20200408132156-9ee5ef7a2c0d // indirect
+	google.golang.org/api v0.21.0 // indirect
+	google.golang.org/genproto v0.0.0-20200408120641-fbb3ad325eb7 // indirect
+	google.golang.org/grpc v1.28.1 // indirect
+)