From e50ae668ede021f4cacd1e55582c7f1bdbebb1b2 Mon Sep 17 00:00:00 2001 From: kristijorgji Date: Wed, 28 Oct 2020 19:00:31 +0100 Subject: [PATCH] allow custom named seeds, make it possible for programmatic usage like for testing or creating another custom seed main --- README.md | 138 ++++++++++++++++++++++++++++++++++++++++++++++--- config.go | 31 +++++++++++ helpers.go | 8 ++- seeder.go | 70 ++++++++++++++++++------- seeder_test.go | 2 +- sources.go | 5 ++ 6 files changed, 224 insertions(+), 30 deletions(-) create mode 100644 config.go diff --git a/README.md b/README.md index e3d6698..99d1a29 100644 --- a/README.md +++ b/README.md @@ -17,17 +17,21 @@ Knowing that this is such an important key element of any big project for testin For now the library supports only MySql as a database driver for its utility functions like `FromJson` provided by `Seeder` struct, but it is db agnostic for your custom seeders you can use any database that is supported by `sql.DB` `goseeder` -1. Allows specifying seeds for different environments such as test,common (all envs) and more (flexible you can define them) -2. Provides out of the box functions like `(s Seeder) FromJson` to seed the table from json data and more data formats and drivers coming soon +1. It is designed for different kind of usages, for both programmatically or building into your exe and run via cli args +1. Allows specifying seeds for different environments such as predefined test and custom specified envs by the user +2. Allows specifying list (or single) seed name for execution +3. Allows having common seeds that execute for every env, unless specified not to do so with the respective cli or option +4. Provides out of the box functions like `(s Seeder) FromJson` to seed the table from json data and more data formats and drivers coming soon # Table of Contents - [Installation](#installation) -- [Usage](#usage) +- [Usage method 1: Turn your executable into seedable via cli args](#usage-method-1-turn-your-executable-into-seedable-via-cli-args) - [1. Change Your Main Function](#1-change-your-main-function) - [2. Registering Your Seeds](#2-registering-your-seeds) - [3. Run Seeds Only For Specific Env](#3-run-seeds-only-for-specific-env) - [4. Run Seeds By Name](#4-run-seeds-by-name) +- [Usage method 2: Programmatically](#usage-method-2-programmatically) - [Summary Of Cli Args](#summary-of-cli-args) - [License](#license) @@ -37,7 +41,7 @@ For now the library supports only MySql as a database driver for its utility fun go get github.com/kristijorgji/goseeder ``` -## Usage +## Usage method 1: Turn your executable into seedable via cli args [Please check examples/simpleshop](examples/simpleshop) for a full working separate go project that uses the seeder @@ -185,6 +189,23 @@ If you have a seed registered for another environment, for example a test seed, So the rule is it will always lookup in this pattern `db/seeds/data/[environment]/[specifiedFileName].[type]` +You can also give a seed a custom name, if you do not want the function name to be used by default. +You can register a seed in a fully flexible way like: + +```go +// db/seeds/common.go +package seeds + +import "github.com/kristijorgji/goseeder" + +func init() { + Registration{ + Name: "another_name_for_cat_seeder", + Env: "stage", + }.Complete(categoriesSeeder) +} +``` + ### 3. Run Seeds Only For Specific Env Many times we want to have seeds only for `test` environment, test purpose and want to avoid having thousand of randomly generated rows inserted into production database by mistake! @@ -248,7 +269,15 @@ To run the test seeder above you have to run: go run main.go --gseed --gsenv=test ``` -This will run only the tests registered for the env `test`, all other seeds will get ignored (also those without environment known as `common` seeds)) +This will run only the tests registered for the env `test` and the `common` seeds. A seed is known as common if it is registered without envrionment via `Register` method and has empty string env. + +If you do not want common seeds to get executed, just specify the flag `--gs-skip-common` + +The above call to run only seeds for test `env`, and ignore the common ones then would be: + +```bash +go run main.go --gseed --gsenv=test --gs-skip-common +``` ### 4. Run Seeds By Name @@ -272,6 +301,99 @@ go run main.go --gseeder --gsnames=categoriesSeeder If you want to execute multiple seeds by specifying their names, just use comma separated value like `--gsnames=categoriesSeeder,someOtherSeeder` +## Usage method 2: Programmatically + +`goseeder` is designed to fit all needs for being the best seeding tool. + +That means that you might want to seed data before your unit tests programmatically without using cli args. + +That is straightforward to do with goseeder. +Let us assume we want to test our api that connects to some database in the package `api`, + +The file `api/main_test.go` might look like below: + +```go +//api/main_test.go +package api + +import ( + "database/sql" + _ "db/seeds" // please import your seeds package so they register or register programatically here too if you want before the seeder Execute is called + "log" + "os" + "testing" + "github.com/kristijorgji/goseeder" +) + + +func TestMain(m *testing.M) { + con := db.ConnectToTestDb() + SeedTestData(con) + r := m.Run() + os.Exit(r) +} + +func SeedTestData(con *sql.DB) { + log.Println("Seeding test database") + goseeder.SetDataPath("../db/seeds/data") + err := goseeder.Execute(con, goseeder.ForEnv("test"), goseeder.ShouldSkipCommon(true)) + if err != nil { + log.Fatal("Seeding test data failed\n") + os.Exit(-2) + } +} +``` + +How nice is that ?! + +After the execution you have a database with data sourcing only from your seeds registered for test env (test seeds) !!! +The above is production code used by one company, but you might need to adjust to your needs. + +Another common use case is to want to execute programmatically the seeder because you don't want to turn your executable into seedable (you don't want to use method 1)). + +Then again you can just create another file `myseeder.go` and inside it do your custom logic or handling of args then just execute +`goseeder.Execute` + +Your `myseeder.go` might look like +```go +package main + +import ( + _ "db/seeds" + "github.com/kristijorgji/goseeder" +) + +func main() { + err := goseeder.Execute(connectToDbOrDie()) + if err != nil { + log.Fatal("Seeding test data failed\n") + os.Exit(-2) + } +} + +// your func here to connect to db +// connectToDbOrDie + +``` + +Then you have your server or app executable separate for example in `main.go` file, and the seeder functionality separated in `myseeder.go` + +You can easily run your seeder `go run myseeder.go`, or build and run etc based on your requirements. + +You can pass all the necessary options to the `goSeeder.Execute` method. +If you want to execute seeders for a particular env only (skip common seeds) for example you do it like: +```go +goseeder.Execute(con, goseeder.ForEnv("test"), goseeder.ShouldSkipCommon(true)) +``` + +These are the possible options you can give after the mandatory db connection: +- `ForEnv(env string)` - you can specify here the env for which you want to execute +- `ForSpecificSeeds(seedNames []string)` - just specify array of seed names you want to execute +- `ShouldSkipCommon(value bool)` - this option has effect only if also gsenv if set, then will not run the common seeds (seeds that do not have any env specified +- `ShouldSkipCommon(value bool)` - this option has effect only if also gsenv if set, then will not run the common seeds (seeds that do not have any env specified + + + ## Summary Of Cli Args You can always run @@ -285,17 +407,17 @@ For the current version the result is: ```bash INR00009:simpleshop kristi.jorgji$ go run main.go --help -Usage of /var/folders/rd/2bkszcpx6xgcddpn7f3bhczn1m9fb7/T/go-build810543742/b001/exe/main: +Usage of /var/folders/rd/2bkszcpx6xgcddpn7f3bhczn1m9fb7/T/go-build358407825/b001/exe/main: + -gs-skip-common + goseeder - this arg has effect only if also gsenv if set, then will not run the common seeds (seeds that do not have any env specified) -gseed goseeder - if set will seed -gsenv string goseeder - env for which seeds to execute -gsnames string goseeder - comma separated seeder names to run specific ones -INR00009:simpleshop kristi.jorgji$ ``` - ## License goseeder is released under the MIT Licence. See the bundled LICENSE file for details. diff --git a/config.go b/config.go new file mode 100644 index 0000000..7a56320 --- /dev/null +++ b/config.go @@ -0,0 +1,31 @@ +package goseeder + +//ConfigOption option to configure the seeder +type ConfigOption = func(config *config) + +type config struct { + env string + seedMethodNames []string + skipCommon bool +} + +//ForEnv specify the env for which the seeders run for +func ForEnv(env string) ConfigOption { + return func(config *config) { + config.env = env + } +} + +//ForSpecificSeeds array of seed names you want to specify for execution +func ForSpecificSeeds(seedNames []string) ConfigOption { + return func(config *config) { + config.seedMethodNames = seedNames + } +} + +//ShouldSkipCommon this option has effect only if also gsenv if set, then will not run the common seeds (seeds that do not have any env specified +func ShouldSkipCommon(value bool) ConfigOption { + return func(config *config) { + config.skipCommon = value + } +} diff --git a/helpers.go b/helpers.go index 14d51e4..be413f4 100644 --- a/helpers.go +++ b/helpers.go @@ -16,7 +16,13 @@ const ( debugColor = "\033[0;36m%s\033[0m" ) -var printError = color(errorColor) +var printInfo = func(s string) { + fmt.Print(color(infoColor)(s)) +} + +var printError = func(s string) { + fmt.Print(color(errorColor)(s)) +} func color(colorString string) func(...interface{}) string { return func(args ...interface{}) string { diff --git a/seeder.go b/seeder.go index dc93588..f3e5c2c 100644 --- a/seeder.go +++ b/seeder.go @@ -14,7 +14,7 @@ import ( // Seeder root seeder offering access to db connection and util functions type Seeder struct { DB *sql.DB - context clientSeeder + context *clientSeeder } //Registration this allows for custom registration with full options available at once, like specifying custom seed name and env in one go. Then have to finish registration by calling Complete @@ -52,11 +52,13 @@ var seeders []clientSeeder func WithSeeder(conProvider func() *sql.DB, clientMain func()) { var seed = false var env = "" + var skipCommon = false var names = "" flag.BoolVar(&seed, "gseed", seed, "goseeder - if set will seed") flag.StringVar(&env, "gsenv", "", "goseeder - env for which seeds to execute") flag.StringVar(&names, "gsnames", "", "goseeder - comma separated seeder names to run specific ones") + flag.BoolVar(&skipCommon, "gs-skip-common", skipCommon, "goseeder - this arg has effect only if also gsenv if set, then will not run the common seeds (seeds that do not have any env specified)") flag.Parse() if !seed { @@ -69,7 +71,16 @@ func WithSeeder(conProvider func() *sql.DB, clientMain func()) { specifiedSeeders = strings.Split(names, ",") } - Execute(conProvider(), env, specifiedSeeders) + err := Execute( + conProvider(), + ForEnv(env), + ForSpecificSeeds(specifiedSeeders), + ShouldSkipCommon(skipCommon), + ) + + if err != nil { + log.Panic(err) + } } // Register the given seed function as common to run for all environments @@ -95,49 +106,68 @@ func RegisterForEnv(env string, seeder func(s Seeder)) { } // Execute use this method for using this lib programmatically and executing -// seeder directly with full flexibility. Be sure first to have registered your -// seeders -func Execute(db *sql.DB, env string, seedMethodNames []string) { +// seeder directly with full flexibility. Be sure first to have registered your seeders +func Execute(db *sql.DB, options ...ConfigOption) error { + c := config{ + env: "", + seedMethodNames: []string{}, + skipCommon: false, + } + for _, option := range options { + option(&c) + } + // Execute all seeders if no method name is given - if len(seedMethodNames) == 0 { - if env == "" { - log.Println("Running all seeders...") + if len(c.seedMethodNames) == 0 { + if c.env == "" { + printInfo(fmt.Sprintf("Running all seeders...\n\n")) } else { - log.Printf("Running seeders for env %s...\n", env) + printInfo(fmt.Sprintf("Running all seeders for env %s and common seeds (without env)...\n\n", c.env)) } for _, seeder := range seeders { - if env == "" || env == seeder.env { - seed(&Seeder{ + if c.env == "" || c.env == seeder.env || (seeder.env == "" && c.skipCommon == false) { + err := seed(&Seeder{ DB: db, - context: seeder, + context: &seeder, }) + if err != nil { + return err + } } } - return + return nil } for _, seeder := range seeders { - if _, r := findString(seedMethodNames, seeder.name); (env == "" || env == seeder.env) && r { - seed(&Seeder{ + if _, r := findString(c.seedMethodNames, seeder.name); (c.env == "" || c.env == seeder.env) && r { + err := seed(&Seeder{ DB: db, - context: seeder, + context: &seeder, }) + if err != nil { + return err + } } } + + return nil } -func seed(seeder *Seeder) { +func seed(seeder *Seeder) (rerr error) { clientSeeder := seeder.context start := time.Now() - log.Printf("[%s] started seeding...\n", clientSeeder.name) + printInfo(fmt.Sprintf("[%s] started seeding...\n", clientSeeder.name)) defer func() { if r := recover(); r != nil { - log.Print(printError(fmt.Sprintf("[%s] seed failed: %+v\n", clientSeeder.name, r))) + msg := fmt.Sprintf("[%s] seed failed: %+v\n", clientSeeder.name, r) + printError(msg) + rerr = errors.New(msg) } }() clientSeeder.cb(*seeder) elapsed := time.Since(start) - log.Printf("[%s] seeded successfully, duration %s\n", clientSeeder.name, elapsed) + printInfo(fmt.Sprintf("[%s] seeded successfully, duration %s\n\n", clientSeeder.name, elapsed)) + return nil } diff --git a/seeder_test.go b/seeder_test.go index 75eec2d..c1e4241 100644 --- a/seeder_test.go +++ b/seeder_test.go @@ -127,7 +127,7 @@ func TestWithSeeder_seed_for_env(t *testing.T) { WithSeeder(conProvider, clientMain) require.Equal(t, 0, mockFnStatus.callCount) - require.Equal(t, 0, mockCommonSeedStatus.callCount) + require.Equal(t, 1, mockCommonSeedStatus.callCount) require.Equal(t, 1, mockSeedSecretStatus.callCount) require.Equal(t, 0, mockSeedStageStatus.callCount) diff --git a/sources.go b/sources.go index a9c53dd..8a4cee6 100644 --- a/sources.go +++ b/sources.go @@ -10,6 +10,11 @@ import ( var dataPath = "db/seeds/data" +//SetDataPath this will allow you to specify a custom path where your seed data is located +func SetDataPath(path string) { + dataPath = path +} + //FromJson inserts into a database table with name same as the filename all the json entries func (s Seeder) FromJson(filename string) { var folder = ""