From a5039769d507c6a4e14021a7060bbb6ab872239b Mon Sep 17 00:00:00 2001 From: Sukhesh Halemane Date: Wed, 5 Aug 2015 20:18:36 -0700 Subject: [PATCH 1/4] Integrate contivModel to netmaster Dont pull docker images if it already exists First version of dockplugin First working version of dockplugin Fix golint issues --- Godeps/Godeps.json | 6 +- .../contiv/objmodel/contivModel/app.json | 30 + .../contivModel/cmExample/apiController.go | 556 ++++ .../objmodel/contivModel/contivModel.go | 2947 +++++++++++++++++ .../objmodel/contivModel/contivModel.js | 4 + .../objmodel/contivModel/endpointGroup.json | 43 + .../contiv/objmodel/contivModel/generate.sh | 1 + .../contiv/objmodel/contivModel/network.json | 53 + .../contiv/objmodel/contivModel/policy.json | 34 + .../contiv/objmodel/contivModel/rule.json | 55 + .../contiv/objmodel/contivModel/service.json | 73 + .../objmodel/contivModel/serviceInstance.json | 40 + .../contiv/objmodel/contivModel/tenant.json | 54 + .../contiv/objmodel/contivModel/volume.json | 42 + .../objmodel/contivModel/volumeProfile.json | 42 + .../contiv/objmodel/objdb/modeldb/modeldb.go | 116 + .../contiv/objmodel/objdb/objdbApi.go | 134 + .../objmodel/objdb/objdbClient/objdbClient.go | 26 + .../objdb/objdbClient/objdbClient_test.go | 310 ++ .../objdb/plugins/etcdClient/etcdClient.go | 211 ++ .../objdb/plugins/etcdClient/etcdLock.go | 357 ++ .../objdb/plugins/etcdClient/etcdService.go | 224 ++ .../contiv/objmodel/objdb/plugins/plugins.go | 10 + drivers/ovsSwitch.go | 108 +- drivers/ovsdriver.go | 41 +- mgmtfn/dockplugin/dockplugin.go | 541 +++ .../dockplugin/libnetClient/libnetClient.go | 574 ++++ netmaster/main.go | 8 +- netmaster/master/netmaster.go | 186 +- netmaster/objApi/apiController.go | 671 ++++ netplugin/netd.go | 5 + scripts/dockerhost/start-dockerhosts | 16 +- 32 files changed, 7403 insertions(+), 115 deletions(-) create mode 100644 Godeps/_workspace/src/github.com/contiv/objmodel/contivModel/app.json create mode 100644 Godeps/_workspace/src/github.com/contiv/objmodel/contivModel/cmExample/apiController.go create mode 100644 Godeps/_workspace/src/github.com/contiv/objmodel/contivModel/contivModel.go create mode 100644 Godeps/_workspace/src/github.com/contiv/objmodel/contivModel/contivModel.js create mode 100644 Godeps/_workspace/src/github.com/contiv/objmodel/contivModel/endpointGroup.json create mode 100644 Godeps/_workspace/src/github.com/contiv/objmodel/contivModel/generate.sh create mode 100644 Godeps/_workspace/src/github.com/contiv/objmodel/contivModel/network.json create mode 100644 Godeps/_workspace/src/github.com/contiv/objmodel/contivModel/policy.json create mode 100644 Godeps/_workspace/src/github.com/contiv/objmodel/contivModel/rule.json create mode 100644 Godeps/_workspace/src/github.com/contiv/objmodel/contivModel/service.json create mode 100644 Godeps/_workspace/src/github.com/contiv/objmodel/contivModel/serviceInstance.json create mode 100644 Godeps/_workspace/src/github.com/contiv/objmodel/contivModel/tenant.json create mode 100644 Godeps/_workspace/src/github.com/contiv/objmodel/contivModel/volume.json create mode 100644 Godeps/_workspace/src/github.com/contiv/objmodel/contivModel/volumeProfile.json create mode 100644 Godeps/_workspace/src/github.com/contiv/objmodel/objdb/modeldb/modeldb.go create mode 100644 Godeps/_workspace/src/github.com/contiv/objmodel/objdb/objdbApi.go create mode 100644 Godeps/_workspace/src/github.com/contiv/objmodel/objdb/objdbClient/objdbClient.go create mode 100644 Godeps/_workspace/src/github.com/contiv/objmodel/objdb/objdbClient/objdbClient_test.go create mode 100644 Godeps/_workspace/src/github.com/contiv/objmodel/objdb/plugins/etcdClient/etcdClient.go create mode 100644 Godeps/_workspace/src/github.com/contiv/objmodel/objdb/plugins/etcdClient/etcdLock.go create mode 100644 Godeps/_workspace/src/github.com/contiv/objmodel/objdb/plugins/etcdClient/etcdService.go create mode 100644 Godeps/_workspace/src/github.com/contiv/objmodel/objdb/plugins/plugins.go create mode 100644 mgmtfn/dockplugin/dockplugin.go create mode 100644 mgmtfn/dockplugin/libnetClient/libnetClient.go create mode 100644 netmaster/objApi/apiController.go diff --git a/Godeps/Godeps.json b/Godeps/Godeps.json index db726f67d..82aa949f3 100644 --- a/Godeps/Godeps.json +++ b/Godeps/Godeps.json @@ -1,6 +1,6 @@ { "ImportPath": "github.com/contiv/netplugin", - "GoVersion": "go1.4.2", + "GoVersion": "go1.4", "Packages": [ "./..." ], @@ -29,6 +29,10 @@ "Comment": "v0.1-3-g45476f1", "Rev": "45476f107954c5e22cf114211e684a5dd9a69595" }, + { + "ImportPath": "github.com/contiv/objmodel/contivModel", + "Rev": "dd43e14dc02b8e0d85ac8469d00beac5386af16d" + }, { "ImportPath": "github.com/contiv/ofnet", "Rev": "2ed83a9c24b58c6bbdb807a771947958ad352bbd" diff --git a/Godeps/_workspace/src/github.com/contiv/objmodel/contivModel/app.json b/Godeps/_workspace/src/github.com/contiv/objmodel/contivModel/app.json new file mode 100644 index 000000000..b071c7787 --- /dev/null +++ b/Godeps/_workspace/src/github.com/contiv/objmodel/contivModel/app.json @@ -0,0 +1,30 @@ +{ + "name": "contivModel", + "objects": [ + { + "name": "app", + "type": "object", + "key": [ "tenantName", "appName" ], + "properties": { + "appName": { + "type": "string", + "description": "Application Name" + }, + "tenantName": { + "type": "string", + "description": "Tenant Name" + } + }, + "link-sets": { + "services": { + "ref": "service" + } + }, + "links": { + "tenant": { + "ref": "tenant" + } + } + } + ] +} diff --git a/Godeps/_workspace/src/github.com/contiv/objmodel/contivModel/cmExample/apiController.go b/Godeps/_workspace/src/github.com/contiv/objmodel/contivModel/cmExample/apiController.go new file mode 100644 index 000000000..0cde2fb80 --- /dev/null +++ b/Godeps/_workspace/src/github.com/contiv/objmodel/contivModel/cmExample/apiController.go @@ -0,0 +1,556 @@ +/*** +Copyright 2014 Cisco Systems Inc. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "errors" + "fmt" + "net/http" + "strconv" + + "github.com/contiv/objmodel/contivModel" + "github.com/contiv/objmodel/objdb/modeldb" + + log "github.com/Sirupsen/logrus" + "github.com/gorilla/mux" +) + +type ApiController struct { + router *mux.Router +} + +var apiCtrler *ApiController + +// Create a new controller +func NewApiController(router *mux.Router) *ApiController { + ctrler := new(ApiController) + ctrler.router = router + + // initialize the model objects + contivModel.Init() + + // Register Callbacks + contivModel.RegisterAppCallbacks(ctrler) + contivModel.RegisterEndpointGroupCallbacks(ctrler) + contivModel.RegisterNetworkCallbacks(ctrler) + contivModel.RegisterPolicyCallbacks(ctrler) + contivModel.RegisterRuleCallbacks(ctrler) + contivModel.RegisterServiceCallbacks(ctrler) + contivModel.RegisterServiceInstanceCallbacks(ctrler) + contivModel.RegisterTenantCallbacks(ctrler) + contivModel.RegisterVolumeCallbacks(ctrler) + contivModel.RegisterVolumeProfileCallbacks(ctrler) + + // Register routes + contivModel.AddRoutes(router) + + // Add default tenant if it doesnt exist + tenant := contivModel.FindTenant("default") + if tenant == nil { + err := contivModel.CreateTenant(&contivModel.Tenant{ + Key: "default", + TenantName: "default", + }) + if err != nil { + log.Fatalf("Error creating default tenant. Err: %v", err) + } + } + + return ctrler +} + +func (self *ApiController) AppCreate(app *contivModel.App) error { + log.Infof("Received AppCreate: %+v", app) + + // Make sure tenant exists + if app.TenantName == "" { + return errors.New("Invalid tenant name") + } + + tenant := contivModel.FindTenant(app.TenantName) + if tenant == nil { + return errors.New("Tenant not found") + } + + // Setup links + modeldb.AddLink(&app.Links.Tenant, tenant) + modeldb.AddLinkSet(&tenant.LinkSets.Apps, app) + + // Save the tenant too since we added the links + err := tenant.Write() + if err != nil { + log.Errorf("Error updating tenant state(%+v). Err: %v", tenant, err) + return err + } + + return nil +} +func (self *ApiController) AppUpdate(app, params *contivModel.App) error { + log.Infof("Received AppUpdate: %+v, params: %+v", app, params) + + return nil +} + +func (self *ApiController) AppDelete(app *contivModel.App) error { + log.Infof("Received AppDelete: %+v", app) + return nil +} + +func (self *ApiController) EndpointGroupCreate(endpointGroup *contivModel.EndpointGroup) error { + log.Infof("Received EndpointGroupCreate: %+v", endpointGroup) + return nil +} + +func (self *ApiController) EndpointGroupUpdate(endpointGroup, params *contivModel.EndpointGroup) error { + log.Infof("Received EndpointGroupUpdate: %+v, params: %+v", endpointGroup, params) + return nil +} + +func (self *ApiController) EndpointGroupDelete(endpointGroup *contivModel.EndpointGroup) error { + log.Infof("Received EndpointGroupDelete: %+v", endpointGroup) + return nil +} + +func (self *ApiController) NetworkCreate(network *contivModel.Network) error { + log.Infof("Received NetworkCreate: %+v", network) + + // Make sure tenant exists + if network.TenantName == "" { + return errors.New("Invalid tenant name") + } + + tenant := contivModel.FindTenant(network.TenantName) + if tenant == nil { + return errors.New("Tenant not found") + } + + // Setup links + modeldb.AddLink(&network.Links.Tenant, tenant) + modeldb.AddLinkSet(&tenant.LinkSets.Networks, network) + + // Save the tenant too since we added the links + err := tenant.Write() + if err != nil { + log.Errorf("Error updating tenant state(%+v). Err: %v", tenant, err) + return err + } + + return nil +} + +func (self *ApiController) NetworkUpdate(network, params *contivModel.Network) error { + log.Infof("Received NetworkUpdate: %+v, params: %+v", network, params) + return nil +} + +func (self *ApiController) NetworkDelete(network *contivModel.Network) error { + log.Infof("Received NetworkDelete: %+v", network) + return nil +} +func (self *ApiController) PolicyCreate(policy *contivModel.Policy) error { + log.Infof("Received PolicyCreate: %+v", policy) + return nil +} +func (self *ApiController) PolicyUpdate(policy, params *contivModel.Policy) error { + log.Infof("Received PolicyUpdate: %+v, params: %+v", policy, params) + return nil +} +func (self *ApiController) PolicyDelete(policy *contivModel.Policy) error { + log.Infof("Received PolicyDelete: %+v", policy) + return nil +} + +func (self *ApiController) RuleCreate(rule *contivModel.Rule) error { + log.Infof("Received RuleCreate: %+v", rule) + return nil +} +func (self *ApiController) RuleUpdate(rule, params *contivModel.Rule) error { + log.Infof("Received RuleUpdate: %+v, params: %+v", rule, params) + return nil +} +func (self *ApiController) RuleDelete(rule *contivModel.Rule) error { + log.Infof("Received RuleDelete: %+v", rule) + return nil +} + +func (self *ApiController) ServiceCreate(service *contivModel.Service) error { + log.Infof("Received ServiceCreate: %+v", service) + + // check params + if (service.TenantName == "") || (service.AppName == "") { + return errors.New("Invalid parameters") + } + + // Make sure tenant exists + tenant := contivModel.FindTenant(service.TenantName) + if tenant == nil { + return errors.New("Tenant not found") + } + + // Find the app this service belongs to + app := contivModel.FindApp(service.TenantName + ":" + service.AppName) + if app == nil { + return errors.New("App not found") + } + + // Setup links + modeldb.AddLink(&service.Links.App, app) + modeldb.AddLinkSet(&app.LinkSets.Services, service) + + // Save the app too since we added the links + err := app.Write() + if err != nil { + return err + } + + // Check if user specified any networks + if len(service.Networks) == 0 { + service.Networks = append(service.Networks, "privateNet") + } + + // link service with network + for _, netName := range service.Networks { + netKey := service.TenantName + ":" + netName + network := contivModel.FindNetwork(netKey) + if network == nil { + log.Errorf("Service: %s could not find network %s", service.Key, netKey) + return errors.New("Network not found") + } + + // Link the network + modeldb.AddLinkSet(&service.LinkSets.Networks, network) + modeldb.AddLinkSet(&network.LinkSets.Services, service) + + // save the network + err := network.Write() + if err != nil { + return err + } + } + + // Check if user specified any endpoint group for the service + if len(service.EndpointGroups) == 0 { + // Create one default endpointGroup per network + for _, netName := range service.Networks { + // params for default endpoint group + dfltEpgName := service.AppName + "." + service.ServiceName + "." + netName + endpointGroup := contivModel.EndpointGroup{ + Key: service.TenantName + ":" + dfltEpgName, + TenantName: service.TenantName, + NetworkName: netName, + GroupName: dfltEpgName, + } + + // Create default endpoint group for the service + err = contivModel.CreateEndpointGroup(&endpointGroup) + if err != nil { + log.Errorf("Error creating endpoint group: %+v, Err: %v", endpointGroup, err) + return err + } + + // Add the endpoint group to the list + service.EndpointGroups = append(service.EndpointGroups, dfltEpgName) + } + } + + // Link the service and endpoint group + for _, epgName := range service.EndpointGroups { + endpointGroup := contivModel.FindEndpointGroup(service.TenantName + ":" + epgName) + if endpointGroup == nil { + log.Errorf("Error: could not find endpoint group: %s", epgName) + return errors.New("could not find endpointGroup") + } + + // setup links + modeldb.AddLinkSet(&service.LinkSets.EndpointGroups, endpointGroup) + modeldb.AddLinkSet(&endpointGroup.LinkSets.Services, service) + + // save the endpointGroup + err = endpointGroup.Write() + if err != nil { + return err + } + } + + // Check if user specified any volume profile + if service.VolumeProfile == "" { + service.VolumeProfile = "default" + } + + volProfKey := service.TenantName + ":" + service.VolumeProfile + volProfile := contivModel.FindVolumeProfile(volProfKey) + if volProfile == nil { + log.Errorf("Could not find the volume profile: %s", service.VolumeProfile) + return errors.New("VolumeProfile not found") + } + + // fixup default values + if service.Scale == 0 { + service.Scale = 1 + } + + // Create service instances + for idx := int64(0); idx < service.Scale; idx++ { + instId := fmt.Sprintf("%d", idx+1) + var volumes []string + + // Create a volume for each instance based on the profile + if volProfile.DatastoreType != "none" { + instVolName := service.AppName + "." + service.ServiceName + "." + instId + err = contivModel.CreateVolume(&contivModel.Volume{ + Key: service.TenantName + ":" + instVolName, + VolumeName: instVolName, + TenantName: service.TenantName, + DatastoreType: volProfile.DatastoreType, + PoolName: volProfile.PoolName, + Size: volProfile.Size, + MountPoint: volProfile.MountPoint, + }) + if err != nil { + log.Errorf("Error creating volume %s. Err: %v", instVolName, err) + return err + } + volumes = []string{instVolName} + } + + // build instance params + instKey := service.TenantName + ":" + service.AppName + ":" + service.ServiceName + ":" + instId + inst := contivModel.ServiceInstance{ + Key: instKey, + InstanceID: instId, + TenantName: service.TenantName, + AppName: service.AppName, + ServiceName: service.ServiceName, + Volumes: volumes, + } + + // create the instance + err := contivModel.CreateServiceInstance(&inst) + if err != nil { + log.Errorf("Error creating service instance: %+v. Err: %v", inst, err) + return err + } + } + + return nil +} + +func (self *ApiController) ServiceUpdate(service, params *contivModel.Service) error { + log.Infof("Received ServiceUpdate: %+v, params: %+v", service, params) + return nil +} + +func (self *ApiController) ServiceDelete(service *contivModel.Service) error { + log.Infof("Received ServiceDelete: %+v", service) + return nil +} + +func (self *ApiController) ServiceInstanceCreate(serviceInstance *contivModel.ServiceInstance) error { + log.Infof("Received ServiceInstanceCreate: %+v", serviceInstance) + inst := serviceInstance + + // Find the service + serviceKey := inst.TenantName + ":" + inst.AppName + ":" + inst.ServiceName + service := contivModel.FindService(serviceKey) + if service == nil { + log.Errorf("Service %s not found for instance: %+v", serviceKey, inst) + return errors.New("Service not found") + } + + // Add links + modeldb.AddLinkSet(&service.LinkSets.Instances, inst) + modeldb.AddLink(&inst.Links.Service, service) + + // setup links with volumes + for _, volumeName := range inst.Volumes { + // find the volume + volume := contivModel.FindVolume(inst.TenantName + ":" + volumeName) + if volume == nil { + log.Errorf("Could not find colume %s for service: %s", volumeName, inst.Key) + return errors.New("Could not find the volume") + } + + // add Links + modeldb.AddLinkSet(&inst.LinkSets.Volumes, volume) + modeldb.AddLinkSet(&volume.LinkSets.ServiceInstances, inst) + } + + return nil +} + +func (self *ApiController) ServiceInstanceUpdate(serviceInstance, params *contivModel.ServiceInstance) error { + log.Infof("Received ServiceInstanceUpdate: %+v, params: %+v", serviceInstance, params) + return nil +} + +func (self *ApiController) ServiceInstanceDelete(serviceInstance *contivModel.ServiceInstance) error { + log.Infof("Received ServiceInstanceDelete: %+v", serviceInstance) + return nil +} +func (self *ApiController) TenantCreate(tenant *contivModel.Tenant) error { + log.Infof("Received TenantCreate: %+v", tenant) + + if tenant.TenantName == "" { + return errors.New("Invalid tenant name") + } + + // Create private network for the tenant + err := contivModel.CreateNetwork(&contivModel.Network{ + Key: tenant.TenantName + ":" + "privateNet", + IsPublic: false, + IsPrivate: true, + Encap: "vxlan", + Subnet: "10.1.0.0/16", + NetworkName: "privateNet", + TenantName: tenant.TenantName, + }) + if err != nil { + log.Errorf("Error creating privateNet for tenant: %+v. Err: %v", tenant, err) + return err + } + + // Create public network for the tenant + err = contivModel.CreateNetwork(&contivModel.Network{ + Key: tenant.TenantName + ":" + "publicNet", + IsPublic: true, + IsPrivate: false, + Encap: "vlan", + Subnet: "192.168.1.0/24", + NetworkName: "publicNet", + TenantName: tenant.TenantName, + }) + if err != nil { + log.Errorf("Error creating publicNet for tenant: %+v. Err: %v", tenant, err) + return err + } + + // Create a default volume profile for the tenant + err = contivModel.CreateVolumeProfile(&contivModel.VolumeProfile{ + Key: tenant.TenantName + ":" + "default", + VolumeProfileName: "default", + TenantName: tenant.TenantName, + DatastoreType: "none", + PoolName: "", + Size: "", + MountPoint: "", + }) + if err != nil { + log.Errorf("Error creating default volume profile. Err: %v", err) + return err + } + + return nil +} + +func (self *ApiController) TenantUpdate(tenant, params *contivModel.Tenant) error { + log.Infof("Received TenantUpdate: %+v, params: %+v", tenant, params) + return nil +} + +func (self *ApiController) TenantDelete(tenant *contivModel.Tenant) error { + log.Infof("Received TenantDelete: %+v", tenant) + return nil +} + +func (self *ApiController) VolumeCreate(volume *contivModel.Volume) error { + log.Infof("Received VolumeCreate: %+v", volume) + + // Make sure tenant exists + if volume.TenantName == "" { + return errors.New("Invalid tenant name") + } + + tenant := contivModel.FindTenant(volume.TenantName) + if tenant == nil { + return errors.New("Tenant not found") + } + + // Setup links + modeldb.AddLink(&volume.Links.Tenant, tenant) + modeldb.AddLinkSet(&tenant.LinkSets.Volumes, volume) + + // Save the tenant too since we added the links + err := tenant.Write() + if err != nil { + return err + } + + return nil +} + +func (self *ApiController) VolumeUpdate(volume, params *contivModel.Volume) error { + log.Infof("Received VolumeUpdate: %+v, params: %+v", volume, params) + return nil +} + +func (self *ApiController) VolumeDelete(volume *contivModel.Volume) error { + log.Infof("Received VolumeDelete: %+v", volume) + return nil +} + +func (self *ApiController) VolumeProfileCreate(volumeProfile *contivModel.VolumeProfile) error { + log.Infof("Received VolumeProfileCreate: %+v", volumeProfile) + + // Make sure tenant exists + if volumeProfile.TenantName == "" { + return errors.New("Invalid tenant name") + } + tenant := contivModel.FindTenant(volumeProfile.TenantName) + if tenant == nil { + return errors.New("Tenant not found") + } + + // Setup links + modeldb.AddLink(&volumeProfile.Links.Tenant, tenant) + modeldb.AddLinkSet(&tenant.LinkSets.VolumeProfiles, volumeProfile) + + // Save the tenant too since we added the links + err := tenant.Write() + if err != nil { + return err + } + + return nil +} + +func (self *ApiController) VolumeProfileUpdate(volumeProfile, params *contivModel.VolumeProfile) error { + log.Infof("Received VolumeProfileUpdate: %+v, params: %+v", volumeProfile, params) + return nil +} + +func (self *ApiController) VolumeProfileDelete(volumeProfile *contivModel.VolumeProfile) error { + return nil +} + +// Create a HTTP Server and initialize the router +func CreateServer(port int) { + listenAddr := ":" + strconv.Itoa(port) + + // Create a router + router := mux.NewRouter() + + // Create the API controller + apiCtrler = NewApiController(router) + + log.Infof("HTTP server listening on %s", listenAddr) + + // Start the HTTP server + log.Fatal(http.ListenAndServe(listenAddr, router)) +} + +func main() { + CreateServer(8000) +} diff --git a/Godeps/_workspace/src/github.com/contiv/objmodel/contivModel/contivModel.go b/Godeps/_workspace/src/github.com/contiv/objmodel/contivModel/contivModel.go new file mode 100644 index 000000000..78e9da765 --- /dev/null +++ b/Godeps/_workspace/src/github.com/contiv/objmodel/contivModel/contivModel.go @@ -0,0 +1,2947 @@ +// contivModel.go +// This file is auto generated by modelgen tool +// Do not edit this file manually + +package contivModel + +import ( + "encoding/json" + "errors" + "net/http" + "regexp" + + log "github.com/Sirupsen/logrus" + "github.com/contiv/objmodel/objdb/modeldb" + "github.com/gorilla/mux" +) + +type HttpApiFunc func(w http.ResponseWriter, r *http.Request, vars map[string]string) (interface{}, error) + +type App struct { + Key string `json:"key,omitempty"` + TenantName string `json:"tenantName,omitempty"` + AppName string `json:"appName,omitempty"` + LinkSets AppLinkSets `json:"link-sets,omitempty"` + Links AppLinks `json:"links,omitempty"` +} + +type AppLinkSets struct { + Services map[string]modeldb.Link `json:"services,omitempty"` +} + +type AppLinks struct { + Tenant modeldb.Link `json:"tenant,omitempty"` +} + +type EndpointGroup struct { + Key string `json:"key,omitempty"` + GroupName string `json:"groupName,omitempty"` + TenantName string `json:"tenantName,omitempty"` + NetworkName string `json:"networkName,omitempty"` + Policies []string `json:"policies,omitempty"` + LinkSets EndpointGroupLinkSets `json:"link-sets,omitempty"` + Links EndpointGroupLinks `json:"links,omitempty"` +} + +type EndpointGroupLinkSets struct { + Services map[string]modeldb.Link `json:"services,omitempty"` + Policies map[string]modeldb.Link `json:"policies,omitempty"` +} + +type EndpointGroupLinks struct { + Tenant modeldb.Link `json:"tenant,omitempty"` + Network modeldb.Link `json:"network,omitempty"` +} + +type Network struct { + Key string `json:"key,omitempty"` + IsPrivate bool `json:"isPrivate,omitempty"` + Encap string `json:"encap,omitempty"` + Subnet string `json:"subnet,omitempty"` + DefaultGw string `json:"defaultGw,omitempty"` + NetworkName string `json:"networkName,omitempty"` + TenantName string `json:"tenantName,omitempty"` + IsPublic bool `json:"isPublic,omitempty"` + LinkSets NetworkLinkSets `json:"link-sets,omitempty"` + Links NetworkLinks `json:"links,omitempty"` +} + +type NetworkLinkSets struct { + EndpointGroups map[string]modeldb.Link `json:"endpointGroups,omitempty"` + Services map[string]modeldb.Link `json:"services,omitempty"` +} + +type NetworkLinks struct { + Tenant modeldb.Link `json:"tenant,omitempty"` +} + +type Policy struct { + Key string `json:"key,omitempty"` + PolicyName string `json:"policyName,omitempty"` + TenantName string `json:"tenantName,omitempty"` + LinkSets PolicyLinkSets `json:"link-sets,omitempty"` + Links PolicyLinks `json:"links,omitempty"` +} + +type PolicyLinkSets struct { + EndpointGroups map[string]modeldb.Link `json:"endpointGroups,omitempty"` + Rules map[string]modeldb.Link `json:"rules,omitempty"` +} + +type PolicyLinks struct { + Tenant modeldb.Link `json:"tenant,omitempty"` +} + +type Rule struct { + Key string `json:"key,omitempty"` + EndpointGroup string `json:"endpointGroup,omitempty"` + Network string `json:"network,omitempty"` + Protocol string `json:"protocol,omitempty"` + RuleName string `json:"ruleName,omitempty"` + PolicyName string `json:"policyName,omitempty"` + TenantName string `json:"tenantName,omitempty"` + Direction string `json:"direction,omitempty"` + IpAddress string `json:"ipAddress,omitempty"` + Port int64 `json:"port,omitempty"` + LinkSets RuleLinkSets `json:"link-sets,omitempty"` +} + +type RuleLinkSets struct { + Policies map[string]modeldb.Link `json:"policies,omitempty"` +} + +type Service struct { + Key string `json:"key,omitempty"` + Scale int64 `json:"scale,omitempty"` + ServiceName string `json:"serviceName,omitempty"` + AppName string `json:"appName,omitempty"` + ImageName string `json:"imageName,omitempty"` + Cpu string `json:"cpu,omitempty"` + EndpointGroups []string `json:"endpointGroups,omitempty"` + Networks []string `json:"networks,omitempty"` + VolumeProfile string `json:"volumeProfile,omitempty"` + TenantName string `json:"tenantName,omitempty"` + Memory string `json:"memory,omitempty"` + Command string `json:"command,omitempty"` + Environment []string `json:"environment,omitempty"` + LinkSets ServiceLinkSets `json:"link-sets,omitempty"` + Links ServiceLinks `json:"links,omitempty"` +} + +type ServiceLinkSets struct { + Networks map[string]modeldb.Link `json:"networks,omitempty"` + EndpointGroups map[string]modeldb.Link `json:"endpointGroups,omitempty"` + Instances map[string]modeldb.Link `json:"instances,omitempty"` +} + +type ServiceLinks struct { + VolumeProfile modeldb.Link `json:"volumeProfile,omitempty"` + App modeldb.Link `json:"app,omitempty"` +} + +type ServiceInstance struct { + Key string `json:"key,omitempty"` + InstanceID string `json:"instanceId,omitempty"` + TenantName string `json:"tenantName,omitempty"` + AppName string `json:"appName,omitempty"` + ServiceName string `json:"serviceName,omitempty"` + Volumes []string `json:"volumes,omitempty"` + LinkSets ServiceInstanceLinkSets `json:"link-sets,omitempty"` + Links ServiceInstanceLinks `json:"links,omitempty"` +} + +type ServiceInstanceLinkSets struct { + Volumes map[string]modeldb.Link `json:"volumes,omitempty"` +} + +type ServiceInstanceLinks struct { + Service modeldb.Link `json:"service,omitempty"` +} + +type Tenant struct { + Key string `json:"key,omitempty"` + TenantName string `json:"tenantName,omitempty"` + SubnetPool string `json:"subnetPool,omitempty"` + SubnetLen int64 `json:"subnetLen,omitempty"` + Vlans string `json:"vlans,omitempty"` + Vxlans string `json:"vxlans,omitempty"` + LinkSets TenantLinkSets `json:"link-sets,omitempty"` +} + +type TenantLinkSets struct { + EndpointGroups map[string]modeldb.Link `json:"endpointGroups,omitempty"` + Policies map[string]modeldb.Link `json:"policies,omitempty"` + Volumes map[string]modeldb.Link `json:"volumes,omitempty"` + VolumeProfiles map[string]modeldb.Link `json:"volumeProfiles,omitempty"` + Networks map[string]modeldb.Link `json:"networks,omitempty"` + Apps map[string]modeldb.Link `json:"apps,omitempty"` +} + +type Volume struct { + Key string `json:"key,omitempty"` + Size string `json:"size,omitempty"` + MountPoint string `json:"mountPoint,omitempty"` + VolumeName string `json:"volumeName,omitempty"` + TenantName string `json:"tenantName,omitempty"` + DatastoreType string `json:"datastoreType,omitempty"` + PoolName string `json:"poolName,omitempty"` + LinkSets VolumeLinkSets `json:"link-sets,omitempty"` + Links VolumeLinks `json:"links,omitempty"` +} + +type VolumeLinkSets struct { + ServiceInstances map[string]modeldb.Link `json:"serviceInstances,omitempty"` +} + +type VolumeLinks struct { + Tenant modeldb.Link `json:"tenant,omitempty"` +} + +type VolumeProfile struct { + Key string `json:"key,omitempty"` + Size string `json:"size,omitempty"` + MountPoint string `json:"mountPoint,omitempty"` + VolumeProfileName string `json:"volumeProfileName,omitempty"` + TenantName string `json:"tenantName,omitempty"` + DatastoreType string `json:"datastoreType,omitempty"` + PoolName string `json:"poolName,omitempty"` + LinkSets VolumeProfileLinkSets `json:"link-sets,omitempty"` + Links VolumeProfileLinks `json:"links,omitempty"` +} + +type VolumeProfileLinkSets struct { + Services map[string]modeldb.Link `json:"services,omitempty"` +} + +type VolumeProfileLinks struct { + Tenant modeldb.Link `json:"tenant,omitempty"` +} + +type Collections struct { + apps map[string]*App + endpointGroups map[string]*EndpointGroup + networks map[string]*Network + policys map[string]*Policy + rules map[string]*Rule + services map[string]*Service + serviceInstances map[string]*ServiceInstance + tenants map[string]*Tenant + volumes map[string]*Volume + volumeProfiles map[string]*VolumeProfile +} + +var collections Collections + +type AppCallbacks interface { + AppCreate(app *App) error + AppUpdate(app, params *App) error + AppDelete(app *App) error +} + +type EndpointGroupCallbacks interface { + EndpointGroupCreate(endpointGroup *EndpointGroup) error + EndpointGroupUpdate(endpointGroup, params *EndpointGroup) error + EndpointGroupDelete(endpointGroup *EndpointGroup) error +} + +type NetworkCallbacks interface { + NetworkCreate(network *Network) error + NetworkUpdate(network, params *Network) error + NetworkDelete(network *Network) error +} + +type PolicyCallbacks interface { + PolicyCreate(policy *Policy) error + PolicyUpdate(policy, params *Policy) error + PolicyDelete(policy *Policy) error +} + +type RuleCallbacks interface { + RuleCreate(rule *Rule) error + RuleUpdate(rule, params *Rule) error + RuleDelete(rule *Rule) error +} + +type ServiceCallbacks interface { + ServiceCreate(service *Service) error + ServiceUpdate(service, params *Service) error + ServiceDelete(service *Service) error +} + +type ServiceInstanceCallbacks interface { + ServiceInstanceCreate(serviceInstance *ServiceInstance) error + ServiceInstanceUpdate(serviceInstance, params *ServiceInstance) error + ServiceInstanceDelete(serviceInstance *ServiceInstance) error +} + +type TenantCallbacks interface { + TenantCreate(tenant *Tenant) error + TenantUpdate(tenant, params *Tenant) error + TenantDelete(tenant *Tenant) error +} + +type VolumeCallbacks interface { + VolumeCreate(volume *Volume) error + VolumeUpdate(volume, params *Volume) error + VolumeDelete(volume *Volume) error +} + +type VolumeProfileCallbacks interface { + VolumeProfileCreate(volumeProfile *VolumeProfile) error + VolumeProfileUpdate(volumeProfile, params *VolumeProfile) error + VolumeProfileDelete(volumeProfile *VolumeProfile) error +} + +type CallbackHandlers struct { + AppCb AppCallbacks + EndpointGroupCb EndpointGroupCallbacks + NetworkCb NetworkCallbacks + PolicyCb PolicyCallbacks + RuleCb RuleCallbacks + ServiceCb ServiceCallbacks + ServiceInstanceCb ServiceInstanceCallbacks + TenantCb TenantCallbacks + VolumeCb VolumeCallbacks + VolumeProfileCb VolumeProfileCallbacks +} + +var objCallbackHandler CallbackHandlers + +func Init() { + collections.apps = make(map[string]*App) + collections.endpointGroups = make(map[string]*EndpointGroup) + collections.networks = make(map[string]*Network) + collections.policys = make(map[string]*Policy) + collections.rules = make(map[string]*Rule) + collections.services = make(map[string]*Service) + collections.serviceInstances = make(map[string]*ServiceInstance) + collections.tenants = make(map[string]*Tenant) + collections.volumes = make(map[string]*Volume) + collections.volumeProfiles = make(map[string]*VolumeProfile) + + restoreApp() + restoreEndpointGroup() + restoreNetwork() + restorePolicy() + restoreRule() + restoreService() + restoreServiceInstance() + restoreTenant() + restoreVolume() + restoreVolumeProfile() +} + +func RegisterAppCallbacks(handler AppCallbacks) { + objCallbackHandler.AppCb = handler +} + +func RegisterEndpointGroupCallbacks(handler EndpointGroupCallbacks) { + objCallbackHandler.EndpointGroupCb = handler +} + +func RegisterNetworkCallbacks(handler NetworkCallbacks) { + objCallbackHandler.NetworkCb = handler +} + +func RegisterPolicyCallbacks(handler PolicyCallbacks) { + objCallbackHandler.PolicyCb = handler +} + +func RegisterRuleCallbacks(handler RuleCallbacks) { + objCallbackHandler.RuleCb = handler +} + +func RegisterServiceCallbacks(handler ServiceCallbacks) { + objCallbackHandler.ServiceCb = handler +} + +func RegisterServiceInstanceCallbacks(handler ServiceInstanceCallbacks) { + objCallbackHandler.ServiceInstanceCb = handler +} + +func RegisterTenantCallbacks(handler TenantCallbacks) { + objCallbackHandler.TenantCb = handler +} + +func RegisterVolumeCallbacks(handler VolumeCallbacks) { + objCallbackHandler.VolumeCb = handler +} + +func RegisterVolumeProfileCallbacks(handler VolumeProfileCallbacks) { + objCallbackHandler.VolumeProfileCb = handler +} + +// Simple Wrapper for http handlers +func makeHttpHandler(handlerFunc HttpApiFunc) http.HandlerFunc { + // Create a closure and return an anonymous function + return func(w http.ResponseWriter, r *http.Request) { + // Call the handler + resp, err := handlerFunc(w, r, mux.Vars(r)) + if err != nil { + // Log error + log.Errorf("Handler for %s %s returned error: %s", r.Method, r.URL, err) + + // Send HTTP response + http.Error(w, err.Error(), http.StatusInternalServerError) + } else { + // Send HTTP response as Json + err = writeJSON(w, http.StatusOK, resp) + if err != nil { + log.Errorf("Error generating json. Err: %v", err) + } + } + } +} + +// writeJSON: writes the value v to the http response stream as json with standard +// json encoding. +func writeJSON(w http.ResponseWriter, code int, v interface{}) error { + // Set content type as json + w.Header().Set("Content-Type", "application/json") + + // write the HTTP status code + w.WriteHeader(code) + + // Write the Json output + return json.NewEncoder(w).Encode(v) +} + +// Add all routes for REST handlers +func AddRoutes(router *mux.Router) { + var route, listRoute string + + // Register app + route = "/api/apps/{key}/" + listRoute = "/api/apps/" + log.Infof("Registering %s", route) + router.Path(listRoute).Methods("GET").HandlerFunc(makeHttpHandler(httpListApps)) + router.Path(route).Methods("GET").HandlerFunc(makeHttpHandler(httpGetApp)) + router.Path(route).Methods("POST").HandlerFunc(makeHttpHandler(httpCreateApp)) + router.Path(route).Methods("PUT").HandlerFunc(makeHttpHandler(httpCreateApp)) + router.Path(route).Methods("DELETE").HandlerFunc(makeHttpHandler(httpDeleteApp)) + + // Register endpointGroup + route = "/api/endpointGroups/{key}/" + listRoute = "/api/endpointGroups/" + log.Infof("Registering %s", route) + router.Path(listRoute).Methods("GET").HandlerFunc(makeHttpHandler(httpListEndpointGroups)) + router.Path(route).Methods("GET").HandlerFunc(makeHttpHandler(httpGetEndpointGroup)) + router.Path(route).Methods("POST").HandlerFunc(makeHttpHandler(httpCreateEndpointGroup)) + router.Path(route).Methods("PUT").HandlerFunc(makeHttpHandler(httpCreateEndpointGroup)) + router.Path(route).Methods("DELETE").HandlerFunc(makeHttpHandler(httpDeleteEndpointGroup)) + + // Register network + route = "/api/networks/{key}/" + listRoute = "/api/networks/" + log.Infof("Registering %s", route) + router.Path(listRoute).Methods("GET").HandlerFunc(makeHttpHandler(httpListNetworks)) + router.Path(route).Methods("GET").HandlerFunc(makeHttpHandler(httpGetNetwork)) + router.Path(route).Methods("POST").HandlerFunc(makeHttpHandler(httpCreateNetwork)) + router.Path(route).Methods("PUT").HandlerFunc(makeHttpHandler(httpCreateNetwork)) + router.Path(route).Methods("DELETE").HandlerFunc(makeHttpHandler(httpDeleteNetwork)) + + // Register policy + route = "/api/policys/{key}/" + listRoute = "/api/policys/" + log.Infof("Registering %s", route) + router.Path(listRoute).Methods("GET").HandlerFunc(makeHttpHandler(httpListPolicys)) + router.Path(route).Methods("GET").HandlerFunc(makeHttpHandler(httpGetPolicy)) + router.Path(route).Methods("POST").HandlerFunc(makeHttpHandler(httpCreatePolicy)) + router.Path(route).Methods("PUT").HandlerFunc(makeHttpHandler(httpCreatePolicy)) + router.Path(route).Methods("DELETE").HandlerFunc(makeHttpHandler(httpDeletePolicy)) + + // Register rule + route = "/api/rules/{key}/" + listRoute = "/api/rules/" + log.Infof("Registering %s", route) + router.Path(listRoute).Methods("GET").HandlerFunc(makeHttpHandler(httpListRules)) + router.Path(route).Methods("GET").HandlerFunc(makeHttpHandler(httpGetRule)) + router.Path(route).Methods("POST").HandlerFunc(makeHttpHandler(httpCreateRule)) + router.Path(route).Methods("PUT").HandlerFunc(makeHttpHandler(httpCreateRule)) + router.Path(route).Methods("DELETE").HandlerFunc(makeHttpHandler(httpDeleteRule)) + + // Register service + route = "/api/services/{key}/" + listRoute = "/api/services/" + log.Infof("Registering %s", route) + router.Path(listRoute).Methods("GET").HandlerFunc(makeHttpHandler(httpListServices)) + router.Path(route).Methods("GET").HandlerFunc(makeHttpHandler(httpGetService)) + router.Path(route).Methods("POST").HandlerFunc(makeHttpHandler(httpCreateService)) + router.Path(route).Methods("PUT").HandlerFunc(makeHttpHandler(httpCreateService)) + router.Path(route).Methods("DELETE").HandlerFunc(makeHttpHandler(httpDeleteService)) + + // Register serviceInstance + route = "/api/serviceInstances/{key}/" + listRoute = "/api/serviceInstances/" + log.Infof("Registering %s", route) + router.Path(listRoute).Methods("GET").HandlerFunc(makeHttpHandler(httpListServiceInstances)) + router.Path(route).Methods("GET").HandlerFunc(makeHttpHandler(httpGetServiceInstance)) + router.Path(route).Methods("POST").HandlerFunc(makeHttpHandler(httpCreateServiceInstance)) + router.Path(route).Methods("PUT").HandlerFunc(makeHttpHandler(httpCreateServiceInstance)) + router.Path(route).Methods("DELETE").HandlerFunc(makeHttpHandler(httpDeleteServiceInstance)) + + // Register tenant + route = "/api/tenants/{key}/" + listRoute = "/api/tenants/" + log.Infof("Registering %s", route) + router.Path(listRoute).Methods("GET").HandlerFunc(makeHttpHandler(httpListTenants)) + router.Path(route).Methods("GET").HandlerFunc(makeHttpHandler(httpGetTenant)) + router.Path(route).Methods("POST").HandlerFunc(makeHttpHandler(httpCreateTenant)) + router.Path(route).Methods("PUT").HandlerFunc(makeHttpHandler(httpCreateTenant)) + router.Path(route).Methods("DELETE").HandlerFunc(makeHttpHandler(httpDeleteTenant)) + + // Register volume + route = "/api/volumes/{key}/" + listRoute = "/api/volumes/" + log.Infof("Registering %s", route) + router.Path(listRoute).Methods("GET").HandlerFunc(makeHttpHandler(httpListVolumes)) + router.Path(route).Methods("GET").HandlerFunc(makeHttpHandler(httpGetVolume)) + router.Path(route).Methods("POST").HandlerFunc(makeHttpHandler(httpCreateVolume)) + router.Path(route).Methods("PUT").HandlerFunc(makeHttpHandler(httpCreateVolume)) + router.Path(route).Methods("DELETE").HandlerFunc(makeHttpHandler(httpDeleteVolume)) + + // Register volumeProfile + route = "/api/volumeProfiles/{key}/" + listRoute = "/api/volumeProfiles/" + log.Infof("Registering %s", route) + router.Path(listRoute).Methods("GET").HandlerFunc(makeHttpHandler(httpListVolumeProfiles)) + router.Path(route).Methods("GET").HandlerFunc(makeHttpHandler(httpGetVolumeProfile)) + router.Path(route).Methods("POST").HandlerFunc(makeHttpHandler(httpCreateVolumeProfile)) + router.Path(route).Methods("PUT").HandlerFunc(makeHttpHandler(httpCreateVolumeProfile)) + router.Path(route).Methods("DELETE").HandlerFunc(makeHttpHandler(httpDeleteVolumeProfile)) + +} + +// LIST REST call +func httpListApps(w http.ResponseWriter, r *http.Request, vars map[string]string) (interface{}, error) { + log.Debugf("Received httpListApps: %+v", vars) + + list := make([]*App, 0) + for _, obj := range collections.apps { + list = append(list, obj) + } + + // Return the list + return list, nil +} + +// GET REST call +func httpGetApp(w http.ResponseWriter, r *http.Request, vars map[string]string) (interface{}, error) { + log.Debugf("Received httpGetApp: %+v", vars) + + key := vars["key"] + + obj := collections.apps[key] + if obj == nil { + log.Errorf("app %s not found", key) + return nil, errors.New("app not found") + } + + // Return the obj + return obj, nil +} + +// CREATE REST call +func httpCreateApp(w http.ResponseWriter, r *http.Request, vars map[string]string) (interface{}, error) { + log.Debugf("Received httpGetApp: %+v", vars) + + var obj App + key := vars["key"] + + // Get object from the request + err := json.NewDecoder(r.Body).Decode(&obj) + if err != nil { + log.Errorf("Error decoding app create request. Err %v", err) + return nil, err + } + + // set the key + obj.Key = key + + // Create the object + err = CreateApp(&obj) + if err != nil { + log.Errorf("CreateApp error for: %+v. Err: %v", obj, err) + return nil, err + } + + // Return the obj + return obj, nil +} + +// DELETE rest call +func httpDeleteApp(w http.ResponseWriter, r *http.Request, vars map[string]string) (interface{}, error) { + log.Debugf("Received httpDeleteApp: %+v", vars) + + key := vars["key"] + + // Delete the object + err := DeleteApp(key) + if err != nil { + log.Errorf("DeleteApp error for: %s. Err: %v", key, err) + return nil, err + } + + // Return the obj + return key, nil +} + +// Create a app object +func CreateApp(obj *App) error { + // Validate parameters + err := ValidateApp(obj) + if err != nil { + log.Errorf("ValidateApp retruned error for: %+v. Err: %v", obj, err) + return err + } + + // Check if we handle this object + if objCallbackHandler.AppCb == nil { + log.Errorf("No callback registered for app object") + return errors.New("Invalid object type") + } + + // Check if object already exists + if collections.apps[obj.Key] != nil { + // Perform Update callback + err = objCallbackHandler.AppCb.AppUpdate(collections.apps[obj.Key], obj) + if err != nil { + log.Errorf("AppUpdate retruned error for: %+v. Err: %v", obj, err) + return err + } + } else { + // save it in cache + collections.apps[obj.Key] = obj + + // Perform Create callback + err = objCallbackHandler.AppCb.AppCreate(obj) + if err != nil { + log.Errorf("AppCreate retruned error for: %+v. Err: %v", obj, err) + delete(collections.apps, obj.Key) + return err + } + } + + // Write it to modeldb + err = obj.Write() + if err != nil { + log.Errorf("Error saving app %s to db. Err: %v", obj.Key, err) + return err + } + + return nil +} + +// Return a pointer to app from collection +func FindApp(key string) *App { + obj := collections.apps[key] + if obj == nil { + log.Errorf("app %s not found", key) + return nil + } + + return obj +} + +// Delete a app object +func DeleteApp(key string) error { + obj := collections.apps[key] + if obj == nil { + log.Errorf("app %s not found", key) + return errors.New("app not found") + } + + // Check if we handle this object + if objCallbackHandler.AppCb == nil { + log.Errorf("No callback registered for app object") + return errors.New("Invalid object type") + } + + // Perform callback + err := objCallbackHandler.AppCb.AppDelete(obj) + if err != nil { + log.Errorf("AppDelete retruned error for: %+v. Err: %v", obj, err) + return err + } + + // delete it from modeldb + err = obj.Delete() + if err != nil { + log.Errorf("Error deleting app %s. Err: %v", obj.Key, err) + } + + // delete it from cache + delete(collections.apps, key) + + return nil +} + +func (self *App) GetType() string { + return "app" +} + +func (self *App) GetKey() string { + return self.Key +} + +func (self *App) Read() error { + if self.Key == "" { + log.Errorf("Empty key while trying to read app object") + return errors.New("Empty key") + } + + return modeldb.ReadObj("app", self.Key, self) +} + +func (self *App) Write() error { + if self.Key == "" { + log.Errorf("Empty key while trying to Write app object") + return errors.New("Empty key") + } + + return modeldb.WriteObj("app", self.Key, self) +} + +func (self *App) Delete() error { + if self.Key == "" { + log.Errorf("Empty key while trying to Delete app object") + return errors.New("Empty key") + } + + return modeldb.DeleteObj("app", self.Key) +} + +func restoreApp() error { + strList, err := modeldb.ReadAllObj("app") + if err != nil { + log.Errorf("Error reading app list. Err: %v", err) + } + + for _, objStr := range strList { + // Parse the json model + var app App + err = json.Unmarshal([]byte(objStr), &app) + if err != nil { + log.Errorf("Error parsing object %s, Err %v", objStr, err) + return err + } + + // add it to the collection + collections.apps[app.Key] = &app + } + + return nil +} + +// Validate a app object +func ValidateApp(obj *App) error { + // Validate key is correct + keyStr := obj.TenantName + ":" + obj.AppName + if obj.Key != keyStr { + log.Errorf("Expecting App Key: %s. Got: %s", keyStr, obj.Key) + return errors.New("Invalid Key") + } + + // Validate each field + + return nil +} + +// LIST REST call +func httpListEndpointGroups(w http.ResponseWriter, r *http.Request, vars map[string]string) (interface{}, error) { + log.Debugf("Received httpListEndpointGroups: %+v", vars) + + list := make([]*EndpointGroup, 0) + for _, obj := range collections.endpointGroups { + list = append(list, obj) + } + + // Return the list + return list, nil +} + +// GET REST call +func httpGetEndpointGroup(w http.ResponseWriter, r *http.Request, vars map[string]string) (interface{}, error) { + log.Debugf("Received httpGetEndpointGroup: %+v", vars) + + key := vars["key"] + + obj := collections.endpointGroups[key] + if obj == nil { + log.Errorf("endpointGroup %s not found", key) + return nil, errors.New("endpointGroup not found") + } + + // Return the obj + return obj, nil +} + +// CREATE REST call +func httpCreateEndpointGroup(w http.ResponseWriter, r *http.Request, vars map[string]string) (interface{}, error) { + log.Debugf("Received httpGetEndpointGroup: %+v", vars) + + var obj EndpointGroup + key := vars["key"] + + // Get object from the request + err := json.NewDecoder(r.Body).Decode(&obj) + if err != nil { + log.Errorf("Error decoding endpointGroup create request. Err %v", err) + return nil, err + } + + // set the key + obj.Key = key + + // Create the object + err = CreateEndpointGroup(&obj) + if err != nil { + log.Errorf("CreateEndpointGroup error for: %+v. Err: %v", obj, err) + return nil, err + } + + // Return the obj + return obj, nil +} + +// DELETE rest call +func httpDeleteEndpointGroup(w http.ResponseWriter, r *http.Request, vars map[string]string) (interface{}, error) { + log.Debugf("Received httpDeleteEndpointGroup: %+v", vars) + + key := vars["key"] + + // Delete the object + err := DeleteEndpointGroup(key) + if err != nil { + log.Errorf("DeleteEndpointGroup error for: %s. Err: %v", key, err) + return nil, err + } + + // Return the obj + return key, nil +} + +// Create a endpointGroup object +func CreateEndpointGroup(obj *EndpointGroup) error { + // Validate parameters + err := ValidateEndpointGroup(obj) + if err != nil { + log.Errorf("ValidateEndpointGroup retruned error for: %+v. Err: %v", obj, err) + return err + } + + // Check if we handle this object + if objCallbackHandler.EndpointGroupCb == nil { + log.Errorf("No callback registered for endpointGroup object") + return errors.New("Invalid object type") + } + + // Check if object already exists + if collections.endpointGroups[obj.Key] != nil { + // Perform Update callback + err = objCallbackHandler.EndpointGroupCb.EndpointGroupUpdate(collections.endpointGroups[obj.Key], obj) + if err != nil { + log.Errorf("EndpointGroupUpdate retruned error for: %+v. Err: %v", obj, err) + return err + } + } else { + // save it in cache + collections.endpointGroups[obj.Key] = obj + + // Perform Create callback + err = objCallbackHandler.EndpointGroupCb.EndpointGroupCreate(obj) + if err != nil { + log.Errorf("EndpointGroupCreate retruned error for: %+v. Err: %v", obj, err) + delete(collections.endpointGroups, obj.Key) + return err + } + } + + // Write it to modeldb + err = obj.Write() + if err != nil { + log.Errorf("Error saving endpointGroup %s to db. Err: %v", obj.Key, err) + return err + } + + return nil +} + +// Return a pointer to endpointGroup from collection +func FindEndpointGroup(key string) *EndpointGroup { + obj := collections.endpointGroups[key] + if obj == nil { + log.Errorf("endpointGroup %s not found", key) + return nil + } + + return obj +} + +// Delete a endpointGroup object +func DeleteEndpointGroup(key string) error { + obj := collections.endpointGroups[key] + if obj == nil { + log.Errorf("endpointGroup %s not found", key) + return errors.New("endpointGroup not found") + } + + // Check if we handle this object + if objCallbackHandler.EndpointGroupCb == nil { + log.Errorf("No callback registered for endpointGroup object") + return errors.New("Invalid object type") + } + + // Perform callback + err := objCallbackHandler.EndpointGroupCb.EndpointGroupDelete(obj) + if err != nil { + log.Errorf("EndpointGroupDelete retruned error for: %+v. Err: %v", obj, err) + return err + } + + // delete it from modeldb + err = obj.Delete() + if err != nil { + log.Errorf("Error deleting endpointGroup %s. Err: %v", obj.Key, err) + } + + // delete it from cache + delete(collections.endpointGroups, key) + + return nil +} + +func (self *EndpointGroup) GetType() string { + return "endpointGroup" +} + +func (self *EndpointGroup) GetKey() string { + return self.Key +} + +func (self *EndpointGroup) Read() error { + if self.Key == "" { + log.Errorf("Empty key while trying to read endpointGroup object") + return errors.New("Empty key") + } + + return modeldb.ReadObj("endpointGroup", self.Key, self) +} + +func (self *EndpointGroup) Write() error { + if self.Key == "" { + log.Errorf("Empty key while trying to Write endpointGroup object") + return errors.New("Empty key") + } + + return modeldb.WriteObj("endpointGroup", self.Key, self) +} + +func (self *EndpointGroup) Delete() error { + if self.Key == "" { + log.Errorf("Empty key while trying to Delete endpointGroup object") + return errors.New("Empty key") + } + + return modeldb.DeleteObj("endpointGroup", self.Key) +} + +func restoreEndpointGroup() error { + strList, err := modeldb.ReadAllObj("endpointGroup") + if err != nil { + log.Errorf("Error reading endpointGroup list. Err: %v", err) + } + + for _, objStr := range strList { + // Parse the json model + var endpointGroup EndpointGroup + err = json.Unmarshal([]byte(objStr), &endpointGroup) + if err != nil { + log.Errorf("Error parsing object %s, Err %v", objStr, err) + return err + } + + // add it to the collection + collections.endpointGroups[endpointGroup.Key] = &endpointGroup + } + + return nil +} + +// Validate a endpointGroup object +func ValidateEndpointGroup(obj *EndpointGroup) error { + // Validate key is correct + keyStr := obj.TenantName + ":" + obj.GroupName + if obj.Key != keyStr { + log.Errorf("Expecting EndpointGroup Key: %s. Got: %s", keyStr, obj.Key) + return errors.New("Invalid Key") + } + + // Validate each field + + return nil +} + +// LIST REST call +func httpListNetworks(w http.ResponseWriter, r *http.Request, vars map[string]string) (interface{}, error) { + log.Debugf("Received httpListNetworks: %+v", vars) + + list := make([]*Network, 0) + for _, obj := range collections.networks { + list = append(list, obj) + } + + // Return the list + return list, nil +} + +// GET REST call +func httpGetNetwork(w http.ResponseWriter, r *http.Request, vars map[string]string) (interface{}, error) { + log.Debugf("Received httpGetNetwork: %+v", vars) + + key := vars["key"] + + obj := collections.networks[key] + if obj == nil { + log.Errorf("network %s not found", key) + return nil, errors.New("network not found") + } + + // Return the obj + return obj, nil +} + +// CREATE REST call +func httpCreateNetwork(w http.ResponseWriter, r *http.Request, vars map[string]string) (interface{}, error) { + log.Debugf("Received httpGetNetwork: %+v", vars) + + var obj Network + key := vars["key"] + + // Get object from the request + err := json.NewDecoder(r.Body).Decode(&obj) + if err != nil { + log.Errorf("Error decoding network create request. Err %v", err) + return nil, err + } + + // set the key + obj.Key = key + + // Create the object + err = CreateNetwork(&obj) + if err != nil { + log.Errorf("CreateNetwork error for: %+v. Err: %v", obj, err) + return nil, err + } + + // Return the obj + return obj, nil +} + +// DELETE rest call +func httpDeleteNetwork(w http.ResponseWriter, r *http.Request, vars map[string]string) (interface{}, error) { + log.Debugf("Received httpDeleteNetwork: %+v", vars) + + key := vars["key"] + + // Delete the object + err := DeleteNetwork(key) + if err != nil { + log.Errorf("DeleteNetwork error for: %s. Err: %v", key, err) + return nil, err + } + + // Return the obj + return key, nil +} + +// Create a network object +func CreateNetwork(obj *Network) error { + // Validate parameters + err := ValidateNetwork(obj) + if err != nil { + log.Errorf("ValidateNetwork retruned error for: %+v. Err: %v", obj, err) + return err + } + + // Check if we handle this object + if objCallbackHandler.NetworkCb == nil { + log.Errorf("No callback registered for network object") + return errors.New("Invalid object type") + } + + // Check if object already exists + if collections.networks[obj.Key] != nil { + // Perform Update callback + err = objCallbackHandler.NetworkCb.NetworkUpdate(collections.networks[obj.Key], obj) + if err != nil { + log.Errorf("NetworkUpdate retruned error for: %+v. Err: %v", obj, err) + return err + } + } else { + // save it in cache + collections.networks[obj.Key] = obj + + // Perform Create callback + err = objCallbackHandler.NetworkCb.NetworkCreate(obj) + if err != nil { + log.Errorf("NetworkCreate retruned error for: %+v. Err: %v", obj, err) + delete(collections.networks, obj.Key) + return err + } + } + + // Write it to modeldb + err = obj.Write() + if err != nil { + log.Errorf("Error saving network %s to db. Err: %v", obj.Key, err) + return err + } + + return nil +} + +// Return a pointer to network from collection +func FindNetwork(key string) *Network { + obj := collections.networks[key] + if obj == nil { + log.Errorf("network %s not found", key) + return nil + } + + return obj +} + +// Delete a network object +func DeleteNetwork(key string) error { + obj := collections.networks[key] + if obj == nil { + log.Errorf("network %s not found", key) + return errors.New("network not found") + } + + // Check if we handle this object + if objCallbackHandler.NetworkCb == nil { + log.Errorf("No callback registered for network object") + return errors.New("Invalid object type") + } + + // Perform callback + err := objCallbackHandler.NetworkCb.NetworkDelete(obj) + if err != nil { + log.Errorf("NetworkDelete retruned error for: %+v. Err: %v", obj, err) + return err + } + + // delete it from modeldb + err = obj.Delete() + if err != nil { + log.Errorf("Error deleting network %s. Err: %v", obj.Key, err) + } + + // delete it from cache + delete(collections.networks, key) + + return nil +} + +func (self *Network) GetType() string { + return "network" +} + +func (self *Network) GetKey() string { + return self.Key +} + +func (self *Network) Read() error { + if self.Key == "" { + log.Errorf("Empty key while trying to read network object") + return errors.New("Empty key") + } + + return modeldb.ReadObj("network", self.Key, self) +} + +func (self *Network) Write() error { + if self.Key == "" { + log.Errorf("Empty key while trying to Write network object") + return errors.New("Empty key") + } + + return modeldb.WriteObj("network", self.Key, self) +} + +func (self *Network) Delete() error { + if self.Key == "" { + log.Errorf("Empty key while trying to Delete network object") + return errors.New("Empty key") + } + + return modeldb.DeleteObj("network", self.Key) +} + +func restoreNetwork() error { + strList, err := modeldb.ReadAllObj("network") + if err != nil { + log.Errorf("Error reading network list. Err: %v", err) + } + + for _, objStr := range strList { + // Parse the json model + var network Network + err = json.Unmarshal([]byte(objStr), &network) + if err != nil { + log.Errorf("Error parsing object %s, Err %v", objStr, err) + return err + } + + // add it to the collection + collections.networks[network.Key] = &network + } + + return nil +} + +// Validate a network object +func ValidateNetwork(obj *Network) error { + // Validate key is correct + keyStr := obj.TenantName + ":" + obj.NetworkName + if obj.Key != keyStr { + log.Errorf("Expecting Network Key: %s. Got: %s", keyStr, obj.Key) + return errors.New("Invalid Key") + } + + // Validate each field + + defaultGwMatch := regexp.MustCompile("^([0-9]{1,3}?.[0-9]{1,3}?.[0-9]{1,3}?.[0-9]{1,3}?)$") + if defaultGwMatch.MatchString(obj.DefaultGw) == false { + return errors.New("defaultGw string invalid format") + } + + encapMatch := regexp.MustCompile("^(vlan|vxlan)$") + if encapMatch.MatchString(obj.Encap) == false { + return errors.New("encap string invalid format") + } + + if len(obj.NetworkName) > 64 { + return errors.New("networkName string too long") + } + + subnetMatch := regexp.MustCompile("^([0-9]{1,3}?.[0-9]{1,3}?.[0-9]{1,3}?.[0-9]{1,3}?/[0-9]{1,2}?)$") + if subnetMatch.MatchString(obj.Subnet) == false { + return errors.New("subnet string invalid format") + } + + if len(obj.TenantName) > 64 { + return errors.New("tenantName string too long") + } + + return nil +} + +// LIST REST call +func httpListPolicys(w http.ResponseWriter, r *http.Request, vars map[string]string) (interface{}, error) { + log.Debugf("Received httpListPolicys: %+v", vars) + + list := make([]*Policy, 0) + for _, obj := range collections.policys { + list = append(list, obj) + } + + // Return the list + return list, nil +} + +// GET REST call +func httpGetPolicy(w http.ResponseWriter, r *http.Request, vars map[string]string) (interface{}, error) { + log.Debugf("Received httpGetPolicy: %+v", vars) + + key := vars["key"] + + obj := collections.policys[key] + if obj == nil { + log.Errorf("policy %s not found", key) + return nil, errors.New("policy not found") + } + + // Return the obj + return obj, nil +} + +// CREATE REST call +func httpCreatePolicy(w http.ResponseWriter, r *http.Request, vars map[string]string) (interface{}, error) { + log.Debugf("Received httpGetPolicy: %+v", vars) + + var obj Policy + key := vars["key"] + + // Get object from the request + err := json.NewDecoder(r.Body).Decode(&obj) + if err != nil { + log.Errorf("Error decoding policy create request. Err %v", err) + return nil, err + } + + // set the key + obj.Key = key + + // Create the object + err = CreatePolicy(&obj) + if err != nil { + log.Errorf("CreatePolicy error for: %+v. Err: %v", obj, err) + return nil, err + } + + // Return the obj + return obj, nil +} + +// DELETE rest call +func httpDeletePolicy(w http.ResponseWriter, r *http.Request, vars map[string]string) (interface{}, error) { + log.Debugf("Received httpDeletePolicy: %+v", vars) + + key := vars["key"] + + // Delete the object + err := DeletePolicy(key) + if err != nil { + log.Errorf("DeletePolicy error for: %s. Err: %v", key, err) + return nil, err + } + + // Return the obj + return key, nil +} + +// Create a policy object +func CreatePolicy(obj *Policy) error { + // Validate parameters + err := ValidatePolicy(obj) + if err != nil { + log.Errorf("ValidatePolicy retruned error for: %+v. Err: %v", obj, err) + return err + } + + // Check if we handle this object + if objCallbackHandler.PolicyCb == nil { + log.Errorf("No callback registered for policy object") + return errors.New("Invalid object type") + } + + // Check if object already exists + if collections.policys[obj.Key] != nil { + // Perform Update callback + err = objCallbackHandler.PolicyCb.PolicyUpdate(collections.policys[obj.Key], obj) + if err != nil { + log.Errorf("PolicyUpdate retruned error for: %+v. Err: %v", obj, err) + return err + } + } else { + // save it in cache + collections.policys[obj.Key] = obj + + // Perform Create callback + err = objCallbackHandler.PolicyCb.PolicyCreate(obj) + if err != nil { + log.Errorf("PolicyCreate retruned error for: %+v. Err: %v", obj, err) + delete(collections.policys, obj.Key) + return err + } + } + + // Write it to modeldb + err = obj.Write() + if err != nil { + log.Errorf("Error saving policy %s to db. Err: %v", obj.Key, err) + return err + } + + return nil +} + +// Return a pointer to policy from collection +func FindPolicy(key string) *Policy { + obj := collections.policys[key] + if obj == nil { + log.Errorf("policy %s not found", key) + return nil + } + + return obj +} + +// Delete a policy object +func DeletePolicy(key string) error { + obj := collections.policys[key] + if obj == nil { + log.Errorf("policy %s not found", key) + return errors.New("policy not found") + } + + // Check if we handle this object + if objCallbackHandler.PolicyCb == nil { + log.Errorf("No callback registered for policy object") + return errors.New("Invalid object type") + } + + // Perform callback + err := objCallbackHandler.PolicyCb.PolicyDelete(obj) + if err != nil { + log.Errorf("PolicyDelete retruned error for: %+v. Err: %v", obj, err) + return err + } + + // delete it from modeldb + err = obj.Delete() + if err != nil { + log.Errorf("Error deleting policy %s. Err: %v", obj.Key, err) + } + + // delete it from cache + delete(collections.policys, key) + + return nil +} + +func (self *Policy) GetType() string { + return "policy" +} + +func (self *Policy) GetKey() string { + return self.Key +} + +func (self *Policy) Read() error { + if self.Key == "" { + log.Errorf("Empty key while trying to read policy object") + return errors.New("Empty key") + } + + return modeldb.ReadObj("policy", self.Key, self) +} + +func (self *Policy) Write() error { + if self.Key == "" { + log.Errorf("Empty key while trying to Write policy object") + return errors.New("Empty key") + } + + return modeldb.WriteObj("policy", self.Key, self) +} + +func (self *Policy) Delete() error { + if self.Key == "" { + log.Errorf("Empty key while trying to Delete policy object") + return errors.New("Empty key") + } + + return modeldb.DeleteObj("policy", self.Key) +} + +func restorePolicy() error { + strList, err := modeldb.ReadAllObj("policy") + if err != nil { + log.Errorf("Error reading policy list. Err: %v", err) + } + + for _, objStr := range strList { + // Parse the json model + var policy Policy + err = json.Unmarshal([]byte(objStr), &policy) + if err != nil { + log.Errorf("Error parsing object %s, Err %v", objStr, err) + return err + } + + // add it to the collection + collections.policys[policy.Key] = &policy + } + + return nil +} + +// Validate a policy object +func ValidatePolicy(obj *Policy) error { + // Validate key is correct + keyStr := obj.TenantName + ":" + obj.PolicyName + if obj.Key != keyStr { + log.Errorf("Expecting Policy Key: %s. Got: %s", keyStr, obj.Key) + return errors.New("Invalid Key") + } + + // Validate each field + + return nil +} + +// LIST REST call +func httpListRules(w http.ResponseWriter, r *http.Request, vars map[string]string) (interface{}, error) { + log.Debugf("Received httpListRules: %+v", vars) + + list := make([]*Rule, 0) + for _, obj := range collections.rules { + list = append(list, obj) + } + + // Return the list + return list, nil +} + +// GET REST call +func httpGetRule(w http.ResponseWriter, r *http.Request, vars map[string]string) (interface{}, error) { + log.Debugf("Received httpGetRule: %+v", vars) + + key := vars["key"] + + obj := collections.rules[key] + if obj == nil { + log.Errorf("rule %s not found", key) + return nil, errors.New("rule not found") + } + + // Return the obj + return obj, nil +} + +// CREATE REST call +func httpCreateRule(w http.ResponseWriter, r *http.Request, vars map[string]string) (interface{}, error) { + log.Debugf("Received httpGetRule: %+v", vars) + + var obj Rule + key := vars["key"] + + // Get object from the request + err := json.NewDecoder(r.Body).Decode(&obj) + if err != nil { + log.Errorf("Error decoding rule create request. Err %v", err) + return nil, err + } + + // set the key + obj.Key = key + + // Create the object + err = CreateRule(&obj) + if err != nil { + log.Errorf("CreateRule error for: %+v. Err: %v", obj, err) + return nil, err + } + + // Return the obj + return obj, nil +} + +// DELETE rest call +func httpDeleteRule(w http.ResponseWriter, r *http.Request, vars map[string]string) (interface{}, error) { + log.Debugf("Received httpDeleteRule: %+v", vars) + + key := vars["key"] + + // Delete the object + err := DeleteRule(key) + if err != nil { + log.Errorf("DeleteRule error for: %s. Err: %v", key, err) + return nil, err + } + + // Return the obj + return key, nil +} + +// Create a rule object +func CreateRule(obj *Rule) error { + // Validate parameters + err := ValidateRule(obj) + if err != nil { + log.Errorf("ValidateRule retruned error for: %+v. Err: %v", obj, err) + return err + } + + // Check if we handle this object + if objCallbackHandler.RuleCb == nil { + log.Errorf("No callback registered for rule object") + return errors.New("Invalid object type") + } + + // Check if object already exists + if collections.rules[obj.Key] != nil { + // Perform Update callback + err = objCallbackHandler.RuleCb.RuleUpdate(collections.rules[obj.Key], obj) + if err != nil { + log.Errorf("RuleUpdate retruned error for: %+v. Err: %v", obj, err) + return err + } + } else { + // save it in cache + collections.rules[obj.Key] = obj + + // Perform Create callback + err = objCallbackHandler.RuleCb.RuleCreate(obj) + if err != nil { + log.Errorf("RuleCreate retruned error for: %+v. Err: %v", obj, err) + delete(collections.rules, obj.Key) + return err + } + } + + // Write it to modeldb + err = obj.Write() + if err != nil { + log.Errorf("Error saving rule %s to db. Err: %v", obj.Key, err) + return err + } + + return nil +} + +// Return a pointer to rule from collection +func FindRule(key string) *Rule { + obj := collections.rules[key] + if obj == nil { + log.Errorf("rule %s not found", key) + return nil + } + + return obj +} + +// Delete a rule object +func DeleteRule(key string) error { + obj := collections.rules[key] + if obj == nil { + log.Errorf("rule %s not found", key) + return errors.New("rule not found") + } + + // Check if we handle this object + if objCallbackHandler.RuleCb == nil { + log.Errorf("No callback registered for rule object") + return errors.New("Invalid object type") + } + + // Perform callback + err := objCallbackHandler.RuleCb.RuleDelete(obj) + if err != nil { + log.Errorf("RuleDelete retruned error for: %+v. Err: %v", obj, err) + return err + } + + // delete it from modeldb + err = obj.Delete() + if err != nil { + log.Errorf("Error deleting rule %s. Err: %v", obj.Key, err) + } + + // delete it from cache + delete(collections.rules, key) + + return nil +} + +func (self *Rule) GetType() string { + return "rule" +} + +func (self *Rule) GetKey() string { + return self.Key +} + +func (self *Rule) Read() error { + if self.Key == "" { + log.Errorf("Empty key while trying to read rule object") + return errors.New("Empty key") + } + + return modeldb.ReadObj("rule", self.Key, self) +} + +func (self *Rule) Write() error { + if self.Key == "" { + log.Errorf("Empty key while trying to Write rule object") + return errors.New("Empty key") + } + + return modeldb.WriteObj("rule", self.Key, self) +} + +func (self *Rule) Delete() error { + if self.Key == "" { + log.Errorf("Empty key while trying to Delete rule object") + return errors.New("Empty key") + } + + return modeldb.DeleteObj("rule", self.Key) +} + +func restoreRule() error { + strList, err := modeldb.ReadAllObj("rule") + if err != nil { + log.Errorf("Error reading rule list. Err: %v", err) + } + + for _, objStr := range strList { + // Parse the json model + var rule Rule + err = json.Unmarshal([]byte(objStr), &rule) + if err != nil { + log.Errorf("Error parsing object %s, Err %v", objStr, err) + return err + } + + // add it to the collection + collections.rules[rule.Key] = &rule + } + + return nil +} + +// Validate a rule object +func ValidateRule(obj *Rule) error { + // Validate key is correct + keyStr := obj.TenantName + ":" + obj.PolicyName + ":" + obj.RuleName + if obj.Key != keyStr { + log.Errorf("Expecting Rule Key: %s. Got: %s", keyStr, obj.Key) + return errors.New("Invalid Key") + } + + // Validate each field + + directionMatch := regexp.MustCompile("^(in|out)$") + if directionMatch.MatchString(obj.Direction) == false { + return errors.New("direction string invalid format") + } + + if len(obj.EndpointGroup) > 64 { + return errors.New("endpointGroup string too long") + } + + if len(obj.Network) > 64 { + return errors.New("network string too long") + } + + if len(obj.PolicyName) > 64 { + return errors.New("policyName string too long") + } + + if obj.Port > 65535 { + return errors.New("port Value Out of bound") + } + + protocolMatch := regexp.MustCompile("^(tcp|udp|icmp|[0-9]{1,3}?)$") + if protocolMatch.MatchString(obj.Protocol) == false { + return errors.New("protocol string invalid format") + } + + if len(obj.RuleName) > 64 { + return errors.New("ruleName string too long") + } + + if len(obj.TenantName) > 64 { + return errors.New("tenantName string too long") + } + + return nil +} + +// LIST REST call +func httpListServices(w http.ResponseWriter, r *http.Request, vars map[string]string) (interface{}, error) { + log.Debugf("Received httpListServices: %+v", vars) + + list := make([]*Service, 0) + for _, obj := range collections.services { + list = append(list, obj) + } + + // Return the list + return list, nil +} + +// GET REST call +func httpGetService(w http.ResponseWriter, r *http.Request, vars map[string]string) (interface{}, error) { + log.Debugf("Received httpGetService: %+v", vars) + + key := vars["key"] + + obj := collections.services[key] + if obj == nil { + log.Errorf("service %s not found", key) + return nil, errors.New("service not found") + } + + // Return the obj + return obj, nil +} + +// CREATE REST call +func httpCreateService(w http.ResponseWriter, r *http.Request, vars map[string]string) (interface{}, error) { + log.Debugf("Received httpGetService: %+v", vars) + + var obj Service + key := vars["key"] + + // Get object from the request + err := json.NewDecoder(r.Body).Decode(&obj) + if err != nil { + log.Errorf("Error decoding service create request. Err %v", err) + return nil, err + } + + // set the key + obj.Key = key + + // Create the object + err = CreateService(&obj) + if err != nil { + log.Errorf("CreateService error for: %+v. Err: %v", obj, err) + return nil, err + } + + // Return the obj + return obj, nil +} + +// DELETE rest call +func httpDeleteService(w http.ResponseWriter, r *http.Request, vars map[string]string) (interface{}, error) { + log.Debugf("Received httpDeleteService: %+v", vars) + + key := vars["key"] + + // Delete the object + err := DeleteService(key) + if err != nil { + log.Errorf("DeleteService error for: %s. Err: %v", key, err) + return nil, err + } + + // Return the obj + return key, nil +} + +// Create a service object +func CreateService(obj *Service) error { + // Validate parameters + err := ValidateService(obj) + if err != nil { + log.Errorf("ValidateService retruned error for: %+v. Err: %v", obj, err) + return err + } + + // Check if we handle this object + if objCallbackHandler.ServiceCb == nil { + log.Errorf("No callback registered for service object") + return errors.New("Invalid object type") + } + + // Check if object already exists + if collections.services[obj.Key] != nil { + // Perform Update callback + err = objCallbackHandler.ServiceCb.ServiceUpdate(collections.services[obj.Key], obj) + if err != nil { + log.Errorf("ServiceUpdate retruned error for: %+v. Err: %v", obj, err) + return err + } + } else { + // save it in cache + collections.services[obj.Key] = obj + + // Perform Create callback + err = objCallbackHandler.ServiceCb.ServiceCreate(obj) + if err != nil { + log.Errorf("ServiceCreate retruned error for: %+v. Err: %v", obj, err) + delete(collections.services, obj.Key) + return err + } + } + + // Write it to modeldb + err = obj.Write() + if err != nil { + log.Errorf("Error saving service %s to db. Err: %v", obj.Key, err) + return err + } + + return nil +} + +// Return a pointer to service from collection +func FindService(key string) *Service { + obj := collections.services[key] + if obj == nil { + log.Errorf("service %s not found", key) + return nil + } + + return obj +} + +// Delete a service object +func DeleteService(key string) error { + obj := collections.services[key] + if obj == nil { + log.Errorf("service %s not found", key) + return errors.New("service not found") + } + + // Check if we handle this object + if objCallbackHandler.ServiceCb == nil { + log.Errorf("No callback registered for service object") + return errors.New("Invalid object type") + } + + // Perform callback + err := objCallbackHandler.ServiceCb.ServiceDelete(obj) + if err != nil { + log.Errorf("ServiceDelete retruned error for: %+v. Err: %v", obj, err) + return err + } + + // delete it from modeldb + err = obj.Delete() + if err != nil { + log.Errorf("Error deleting service %s. Err: %v", obj.Key, err) + } + + // delete it from cache + delete(collections.services, key) + + return nil +} + +func (self *Service) GetType() string { + return "service" +} + +func (self *Service) GetKey() string { + return self.Key +} + +func (self *Service) Read() error { + if self.Key == "" { + log.Errorf("Empty key while trying to read service object") + return errors.New("Empty key") + } + + return modeldb.ReadObj("service", self.Key, self) +} + +func (self *Service) Write() error { + if self.Key == "" { + log.Errorf("Empty key while trying to Write service object") + return errors.New("Empty key") + } + + return modeldb.WriteObj("service", self.Key, self) +} + +func (self *Service) Delete() error { + if self.Key == "" { + log.Errorf("Empty key while trying to Delete service object") + return errors.New("Empty key") + } + + return modeldb.DeleteObj("service", self.Key) +} + +func restoreService() error { + strList, err := modeldb.ReadAllObj("service") + if err != nil { + log.Errorf("Error reading service list. Err: %v", err) + } + + for _, objStr := range strList { + // Parse the json model + var service Service + err = json.Unmarshal([]byte(objStr), &service) + if err != nil { + log.Errorf("Error parsing object %s, Err %v", objStr, err) + return err + } + + // add it to the collection + collections.services[service.Key] = &service + } + + return nil +} + +// Validate a service object +func ValidateService(obj *Service) error { + // Validate key is correct + keyStr := obj.TenantName + ":" + obj.AppName + ":" + obj.ServiceName + if obj.Key != keyStr { + log.Errorf("Expecting Service Key: %s. Got: %s", keyStr, obj.Key) + return errors.New("Invalid Key") + } + + // Validate each field + + return nil +} + +// LIST REST call +func httpListServiceInstances(w http.ResponseWriter, r *http.Request, vars map[string]string) (interface{}, error) { + log.Debugf("Received httpListServiceInstances: %+v", vars) + + list := make([]*ServiceInstance, 0) + for _, obj := range collections.serviceInstances { + list = append(list, obj) + } + + // Return the list + return list, nil +} + +// GET REST call +func httpGetServiceInstance(w http.ResponseWriter, r *http.Request, vars map[string]string) (interface{}, error) { + log.Debugf("Received httpGetServiceInstance: %+v", vars) + + key := vars["key"] + + obj := collections.serviceInstances[key] + if obj == nil { + log.Errorf("serviceInstance %s not found", key) + return nil, errors.New("serviceInstance not found") + } + + // Return the obj + return obj, nil +} + +// CREATE REST call +func httpCreateServiceInstance(w http.ResponseWriter, r *http.Request, vars map[string]string) (interface{}, error) { + log.Debugf("Received httpGetServiceInstance: %+v", vars) + + var obj ServiceInstance + key := vars["key"] + + // Get object from the request + err := json.NewDecoder(r.Body).Decode(&obj) + if err != nil { + log.Errorf("Error decoding serviceInstance create request. Err %v", err) + return nil, err + } + + // set the key + obj.Key = key + + // Create the object + err = CreateServiceInstance(&obj) + if err != nil { + log.Errorf("CreateServiceInstance error for: %+v. Err: %v", obj, err) + return nil, err + } + + // Return the obj + return obj, nil +} + +// DELETE rest call +func httpDeleteServiceInstance(w http.ResponseWriter, r *http.Request, vars map[string]string) (interface{}, error) { + log.Debugf("Received httpDeleteServiceInstance: %+v", vars) + + key := vars["key"] + + // Delete the object + err := DeleteServiceInstance(key) + if err != nil { + log.Errorf("DeleteServiceInstance error for: %s. Err: %v", key, err) + return nil, err + } + + // Return the obj + return key, nil +} + +// Create a serviceInstance object +func CreateServiceInstance(obj *ServiceInstance) error { + // Validate parameters + err := ValidateServiceInstance(obj) + if err != nil { + log.Errorf("ValidateServiceInstance retruned error for: %+v. Err: %v", obj, err) + return err + } + + // Check if we handle this object + if objCallbackHandler.ServiceInstanceCb == nil { + log.Errorf("No callback registered for serviceInstance object") + return errors.New("Invalid object type") + } + + // Check if object already exists + if collections.serviceInstances[obj.Key] != nil { + // Perform Update callback + err = objCallbackHandler.ServiceInstanceCb.ServiceInstanceUpdate(collections.serviceInstances[obj.Key], obj) + if err != nil { + log.Errorf("ServiceInstanceUpdate retruned error for: %+v. Err: %v", obj, err) + return err + } + } else { + // save it in cache + collections.serviceInstances[obj.Key] = obj + + // Perform Create callback + err = objCallbackHandler.ServiceInstanceCb.ServiceInstanceCreate(obj) + if err != nil { + log.Errorf("ServiceInstanceCreate retruned error for: %+v. Err: %v", obj, err) + delete(collections.serviceInstances, obj.Key) + return err + } + } + + // Write it to modeldb + err = obj.Write() + if err != nil { + log.Errorf("Error saving serviceInstance %s to db. Err: %v", obj.Key, err) + return err + } + + return nil +} + +// Return a pointer to serviceInstance from collection +func FindServiceInstance(key string) *ServiceInstance { + obj := collections.serviceInstances[key] + if obj == nil { + log.Errorf("serviceInstance %s not found", key) + return nil + } + + return obj +} + +// Delete a serviceInstance object +func DeleteServiceInstance(key string) error { + obj := collections.serviceInstances[key] + if obj == nil { + log.Errorf("serviceInstance %s not found", key) + return errors.New("serviceInstance not found") + } + + // Check if we handle this object + if objCallbackHandler.ServiceInstanceCb == nil { + log.Errorf("No callback registered for serviceInstance object") + return errors.New("Invalid object type") + } + + // Perform callback + err := objCallbackHandler.ServiceInstanceCb.ServiceInstanceDelete(obj) + if err != nil { + log.Errorf("ServiceInstanceDelete retruned error for: %+v. Err: %v", obj, err) + return err + } + + // delete it from modeldb + err = obj.Delete() + if err != nil { + log.Errorf("Error deleting serviceInstance %s. Err: %v", obj.Key, err) + } + + // delete it from cache + delete(collections.serviceInstances, key) + + return nil +} + +func (self *ServiceInstance) GetType() string { + return "serviceInstance" +} + +func (self *ServiceInstance) GetKey() string { + return self.Key +} + +func (self *ServiceInstance) Read() error { + if self.Key == "" { + log.Errorf("Empty key while trying to read serviceInstance object") + return errors.New("Empty key") + } + + return modeldb.ReadObj("serviceInstance", self.Key, self) +} + +func (self *ServiceInstance) Write() error { + if self.Key == "" { + log.Errorf("Empty key while trying to Write serviceInstance object") + return errors.New("Empty key") + } + + return modeldb.WriteObj("serviceInstance", self.Key, self) +} + +func (self *ServiceInstance) Delete() error { + if self.Key == "" { + log.Errorf("Empty key while trying to Delete serviceInstance object") + return errors.New("Empty key") + } + + return modeldb.DeleteObj("serviceInstance", self.Key) +} + +func restoreServiceInstance() error { + strList, err := modeldb.ReadAllObj("serviceInstance") + if err != nil { + log.Errorf("Error reading serviceInstance list. Err: %v", err) + } + + for _, objStr := range strList { + // Parse the json model + var serviceInstance ServiceInstance + err = json.Unmarshal([]byte(objStr), &serviceInstance) + if err != nil { + log.Errorf("Error parsing object %s, Err %v", objStr, err) + return err + } + + // add it to the collection + collections.serviceInstances[serviceInstance.Key] = &serviceInstance + } + + return nil +} + +// Validate a serviceInstance object +func ValidateServiceInstance(obj *ServiceInstance) error { + // Validate key is correct + keyStr := obj.TenantName + ":" + obj.AppName + ":" + obj.ServiceName + ":" + obj.InstanceID + if obj.Key != keyStr { + log.Errorf("Expecting ServiceInstance Key: %s. Got: %s", keyStr, obj.Key) + return errors.New("Invalid Key") + } + + // Validate each field + + return nil +} + +// LIST REST call +func httpListTenants(w http.ResponseWriter, r *http.Request, vars map[string]string) (interface{}, error) { + log.Debugf("Received httpListTenants: %+v", vars) + + list := make([]*Tenant, 0) + for _, obj := range collections.tenants { + list = append(list, obj) + } + + // Return the list + return list, nil +} + +// GET REST call +func httpGetTenant(w http.ResponseWriter, r *http.Request, vars map[string]string) (interface{}, error) { + log.Debugf("Received httpGetTenant: %+v", vars) + + key := vars["key"] + + obj := collections.tenants[key] + if obj == nil { + log.Errorf("tenant %s not found", key) + return nil, errors.New("tenant not found") + } + + // Return the obj + return obj, nil +} + +// CREATE REST call +func httpCreateTenant(w http.ResponseWriter, r *http.Request, vars map[string]string) (interface{}, error) { + log.Debugf("Received httpGetTenant: %+v", vars) + + var obj Tenant + key := vars["key"] + + // Get object from the request + err := json.NewDecoder(r.Body).Decode(&obj) + if err != nil { + log.Errorf("Error decoding tenant create request. Err %v", err) + return nil, err + } + + // set the key + obj.Key = key + + // Create the object + err = CreateTenant(&obj) + if err != nil { + log.Errorf("CreateTenant error for: %+v. Err: %v", obj, err) + return nil, err + } + + // Return the obj + return obj, nil +} + +// DELETE rest call +func httpDeleteTenant(w http.ResponseWriter, r *http.Request, vars map[string]string) (interface{}, error) { + log.Debugf("Received httpDeleteTenant: %+v", vars) + + key := vars["key"] + + // Delete the object + err := DeleteTenant(key) + if err != nil { + log.Errorf("DeleteTenant error for: %s. Err: %v", key, err) + return nil, err + } + + // Return the obj + return key, nil +} + +// Create a tenant object +func CreateTenant(obj *Tenant) error { + // Validate parameters + err := ValidateTenant(obj) + if err != nil { + log.Errorf("ValidateTenant retruned error for: %+v. Err: %v", obj, err) + return err + } + + // Check if we handle this object + if objCallbackHandler.TenantCb == nil { + log.Errorf("No callback registered for tenant object") + return errors.New("Invalid object type") + } + + // Check if object already exists + if collections.tenants[obj.Key] != nil { + // Perform Update callback + err = objCallbackHandler.TenantCb.TenantUpdate(collections.tenants[obj.Key], obj) + if err != nil { + log.Errorf("TenantUpdate retruned error for: %+v. Err: %v", obj, err) + return err + } + } else { + // save it in cache + collections.tenants[obj.Key] = obj + + // Perform Create callback + err = objCallbackHandler.TenantCb.TenantCreate(obj) + if err != nil { + log.Errorf("TenantCreate retruned error for: %+v. Err: %v", obj, err) + delete(collections.tenants, obj.Key) + return err + } + } + + // Write it to modeldb + err = obj.Write() + if err != nil { + log.Errorf("Error saving tenant %s to db. Err: %v", obj.Key, err) + return err + } + + return nil +} + +// Return a pointer to tenant from collection +func FindTenant(key string) *Tenant { + obj := collections.tenants[key] + if obj == nil { + log.Errorf("tenant %s not found", key) + return nil + } + + return obj +} + +// Delete a tenant object +func DeleteTenant(key string) error { + obj := collections.tenants[key] + if obj == nil { + log.Errorf("tenant %s not found", key) + return errors.New("tenant not found") + } + + // Check if we handle this object + if objCallbackHandler.TenantCb == nil { + log.Errorf("No callback registered for tenant object") + return errors.New("Invalid object type") + } + + // Perform callback + err := objCallbackHandler.TenantCb.TenantDelete(obj) + if err != nil { + log.Errorf("TenantDelete retruned error for: %+v. Err: %v", obj, err) + return err + } + + // delete it from modeldb + err = obj.Delete() + if err != nil { + log.Errorf("Error deleting tenant %s. Err: %v", obj.Key, err) + } + + // delete it from cache + delete(collections.tenants, key) + + return nil +} + +func (self *Tenant) GetType() string { + return "tenant" +} + +func (self *Tenant) GetKey() string { + return self.Key +} + +func (self *Tenant) Read() error { + if self.Key == "" { + log.Errorf("Empty key while trying to read tenant object") + return errors.New("Empty key") + } + + return modeldb.ReadObj("tenant", self.Key, self) +} + +func (self *Tenant) Write() error { + if self.Key == "" { + log.Errorf("Empty key while trying to Write tenant object") + return errors.New("Empty key") + } + + return modeldb.WriteObj("tenant", self.Key, self) +} + +func (self *Tenant) Delete() error { + if self.Key == "" { + log.Errorf("Empty key while trying to Delete tenant object") + return errors.New("Empty key") + } + + return modeldb.DeleteObj("tenant", self.Key) +} + +func restoreTenant() error { + strList, err := modeldb.ReadAllObj("tenant") + if err != nil { + log.Errorf("Error reading tenant list. Err: %v", err) + } + + for _, objStr := range strList { + // Parse the json model + var tenant Tenant + err = json.Unmarshal([]byte(objStr), &tenant) + if err != nil { + log.Errorf("Error parsing object %s, Err %v", objStr, err) + return err + } + + // add it to the collection + collections.tenants[tenant.Key] = &tenant + } + + return nil +} + +// Validate a tenant object +func ValidateTenant(obj *Tenant) error { + // Validate key is correct + keyStr := obj.TenantName + if obj.Key != keyStr { + log.Errorf("Expecting Tenant Key: %s. Got: %s", keyStr, obj.Key) + return errors.New("Invalid Key") + } + + // Validate each field + + if obj.SubnetLen < 1 { + return errors.New("subnetLen Value Out of bound") + } + + if obj.SubnetLen > 32 { + return errors.New("subnetLen Value Out of bound") + } + + subnetPoolMatch := regexp.MustCompile("^([0-9]{1,3}?.[0-9]{1,3}?.[0-9]{1,3}?.[0-9]{1,3}?/[0-9]{1,2}?)$") + if subnetPoolMatch.MatchString(obj.SubnetPool) == false { + return errors.New("subnetPool string invalid format") + } + + if len(obj.TenantName) > 64 { + return errors.New("tenantName string too long") + } + + vlansMatch := regexp.MustCompile("^([0-9]{1,4}?-[0-9]{1,4}?)$") + if vlansMatch.MatchString(obj.Vlans) == false { + return errors.New("vlans string invalid format") + } + + vxlansMatch := regexp.MustCompile("^([0-9]{1,8}?-[0-9]{1,8}?)$") + if vxlansMatch.MatchString(obj.Vxlans) == false { + return errors.New("vxlans string invalid format") + } + + return nil +} + +// LIST REST call +func httpListVolumes(w http.ResponseWriter, r *http.Request, vars map[string]string) (interface{}, error) { + log.Debugf("Received httpListVolumes: %+v", vars) + + list := make([]*Volume, 0) + for _, obj := range collections.volumes { + list = append(list, obj) + } + + // Return the list + return list, nil +} + +// GET REST call +func httpGetVolume(w http.ResponseWriter, r *http.Request, vars map[string]string) (interface{}, error) { + log.Debugf("Received httpGetVolume: %+v", vars) + + key := vars["key"] + + obj := collections.volumes[key] + if obj == nil { + log.Errorf("volume %s not found", key) + return nil, errors.New("volume not found") + } + + // Return the obj + return obj, nil +} + +// CREATE REST call +func httpCreateVolume(w http.ResponseWriter, r *http.Request, vars map[string]string) (interface{}, error) { + log.Debugf("Received httpGetVolume: %+v", vars) + + var obj Volume + key := vars["key"] + + // Get object from the request + err := json.NewDecoder(r.Body).Decode(&obj) + if err != nil { + log.Errorf("Error decoding volume create request. Err %v", err) + return nil, err + } + + // set the key + obj.Key = key + + // Create the object + err = CreateVolume(&obj) + if err != nil { + log.Errorf("CreateVolume error for: %+v. Err: %v", obj, err) + return nil, err + } + + // Return the obj + return obj, nil +} + +// DELETE rest call +func httpDeleteVolume(w http.ResponseWriter, r *http.Request, vars map[string]string) (interface{}, error) { + log.Debugf("Received httpDeleteVolume: %+v", vars) + + key := vars["key"] + + // Delete the object + err := DeleteVolume(key) + if err != nil { + log.Errorf("DeleteVolume error for: %s. Err: %v", key, err) + return nil, err + } + + // Return the obj + return key, nil +} + +// Create a volume object +func CreateVolume(obj *Volume) error { + // Validate parameters + err := ValidateVolume(obj) + if err != nil { + log.Errorf("ValidateVolume retruned error for: %+v. Err: %v", obj, err) + return err + } + + // Check if we handle this object + if objCallbackHandler.VolumeCb == nil { + log.Errorf("No callback registered for volume object") + return errors.New("Invalid object type") + } + + // Check if object already exists + if collections.volumes[obj.Key] != nil { + // Perform Update callback + err = objCallbackHandler.VolumeCb.VolumeUpdate(collections.volumes[obj.Key], obj) + if err != nil { + log.Errorf("VolumeUpdate retruned error for: %+v. Err: %v", obj, err) + return err + } + } else { + // save it in cache + collections.volumes[obj.Key] = obj + + // Perform Create callback + err = objCallbackHandler.VolumeCb.VolumeCreate(obj) + if err != nil { + log.Errorf("VolumeCreate retruned error for: %+v. Err: %v", obj, err) + delete(collections.volumes, obj.Key) + return err + } + } + + // Write it to modeldb + err = obj.Write() + if err != nil { + log.Errorf("Error saving volume %s to db. Err: %v", obj.Key, err) + return err + } + + return nil +} + +// Return a pointer to volume from collection +func FindVolume(key string) *Volume { + obj := collections.volumes[key] + if obj == nil { + log.Errorf("volume %s not found", key) + return nil + } + + return obj +} + +// Delete a volume object +func DeleteVolume(key string) error { + obj := collections.volumes[key] + if obj == nil { + log.Errorf("volume %s not found", key) + return errors.New("volume not found") + } + + // Check if we handle this object + if objCallbackHandler.VolumeCb == nil { + log.Errorf("No callback registered for volume object") + return errors.New("Invalid object type") + } + + // Perform callback + err := objCallbackHandler.VolumeCb.VolumeDelete(obj) + if err != nil { + log.Errorf("VolumeDelete retruned error for: %+v. Err: %v", obj, err) + return err + } + + // delete it from modeldb + err = obj.Delete() + if err != nil { + log.Errorf("Error deleting volume %s. Err: %v", obj.Key, err) + } + + // delete it from cache + delete(collections.volumes, key) + + return nil +} + +func (self *Volume) GetType() string { + return "volume" +} + +func (self *Volume) GetKey() string { + return self.Key +} + +func (self *Volume) Read() error { + if self.Key == "" { + log.Errorf("Empty key while trying to read volume object") + return errors.New("Empty key") + } + + return modeldb.ReadObj("volume", self.Key, self) +} + +func (self *Volume) Write() error { + if self.Key == "" { + log.Errorf("Empty key while trying to Write volume object") + return errors.New("Empty key") + } + + return modeldb.WriteObj("volume", self.Key, self) +} + +func (self *Volume) Delete() error { + if self.Key == "" { + log.Errorf("Empty key while trying to Delete volume object") + return errors.New("Empty key") + } + + return modeldb.DeleteObj("volume", self.Key) +} + +func restoreVolume() error { + strList, err := modeldb.ReadAllObj("volume") + if err != nil { + log.Errorf("Error reading volume list. Err: %v", err) + } + + for _, objStr := range strList { + // Parse the json model + var volume Volume + err = json.Unmarshal([]byte(objStr), &volume) + if err != nil { + log.Errorf("Error parsing object %s, Err %v", objStr, err) + return err + } + + // add it to the collection + collections.volumes[volume.Key] = &volume + } + + return nil +} + +// Validate a volume object +func ValidateVolume(obj *Volume) error { + // Validate key is correct + keyStr := obj.TenantName + ":" + obj.VolumeName + if obj.Key != keyStr { + log.Errorf("Expecting Volume Key: %s. Got: %s", keyStr, obj.Key) + return errors.New("Invalid Key") + } + + // Validate each field + + return nil +} + +// LIST REST call +func httpListVolumeProfiles(w http.ResponseWriter, r *http.Request, vars map[string]string) (interface{}, error) { + log.Debugf("Received httpListVolumeProfiles: %+v", vars) + + list := make([]*VolumeProfile, 0) + for _, obj := range collections.volumeProfiles { + list = append(list, obj) + } + + // Return the list + return list, nil +} + +// GET REST call +func httpGetVolumeProfile(w http.ResponseWriter, r *http.Request, vars map[string]string) (interface{}, error) { + log.Debugf("Received httpGetVolumeProfile: %+v", vars) + + key := vars["key"] + + obj := collections.volumeProfiles[key] + if obj == nil { + log.Errorf("volumeProfile %s not found", key) + return nil, errors.New("volumeProfile not found") + } + + // Return the obj + return obj, nil +} + +// CREATE REST call +func httpCreateVolumeProfile(w http.ResponseWriter, r *http.Request, vars map[string]string) (interface{}, error) { + log.Debugf("Received httpGetVolumeProfile: %+v", vars) + + var obj VolumeProfile + key := vars["key"] + + // Get object from the request + err := json.NewDecoder(r.Body).Decode(&obj) + if err != nil { + log.Errorf("Error decoding volumeProfile create request. Err %v", err) + return nil, err + } + + // set the key + obj.Key = key + + // Create the object + err = CreateVolumeProfile(&obj) + if err != nil { + log.Errorf("CreateVolumeProfile error for: %+v. Err: %v", obj, err) + return nil, err + } + + // Return the obj + return obj, nil +} + +// DELETE rest call +func httpDeleteVolumeProfile(w http.ResponseWriter, r *http.Request, vars map[string]string) (interface{}, error) { + log.Debugf("Received httpDeleteVolumeProfile: %+v", vars) + + key := vars["key"] + + // Delete the object + err := DeleteVolumeProfile(key) + if err != nil { + log.Errorf("DeleteVolumeProfile error for: %s. Err: %v", key, err) + return nil, err + } + + // Return the obj + return key, nil +} + +// Create a volumeProfile object +func CreateVolumeProfile(obj *VolumeProfile) error { + // Validate parameters + err := ValidateVolumeProfile(obj) + if err != nil { + log.Errorf("ValidateVolumeProfile retruned error for: %+v. Err: %v", obj, err) + return err + } + + // Check if we handle this object + if objCallbackHandler.VolumeProfileCb == nil { + log.Errorf("No callback registered for volumeProfile object") + return errors.New("Invalid object type") + } + + // Check if object already exists + if collections.volumeProfiles[obj.Key] != nil { + // Perform Update callback + err = objCallbackHandler.VolumeProfileCb.VolumeProfileUpdate(collections.volumeProfiles[obj.Key], obj) + if err != nil { + log.Errorf("VolumeProfileUpdate retruned error for: %+v. Err: %v", obj, err) + return err + } + } else { + // save it in cache + collections.volumeProfiles[obj.Key] = obj + + // Perform Create callback + err = objCallbackHandler.VolumeProfileCb.VolumeProfileCreate(obj) + if err != nil { + log.Errorf("VolumeProfileCreate retruned error for: %+v. Err: %v", obj, err) + delete(collections.volumeProfiles, obj.Key) + return err + } + } + + // Write it to modeldb + err = obj.Write() + if err != nil { + log.Errorf("Error saving volumeProfile %s to db. Err: %v", obj.Key, err) + return err + } + + return nil +} + +// Return a pointer to volumeProfile from collection +func FindVolumeProfile(key string) *VolumeProfile { + obj := collections.volumeProfiles[key] + if obj == nil { + log.Errorf("volumeProfile %s not found", key) + return nil + } + + return obj +} + +// Delete a volumeProfile object +func DeleteVolumeProfile(key string) error { + obj := collections.volumeProfiles[key] + if obj == nil { + log.Errorf("volumeProfile %s not found", key) + return errors.New("volumeProfile not found") + } + + // Check if we handle this object + if objCallbackHandler.VolumeProfileCb == nil { + log.Errorf("No callback registered for volumeProfile object") + return errors.New("Invalid object type") + } + + // Perform callback + err := objCallbackHandler.VolumeProfileCb.VolumeProfileDelete(obj) + if err != nil { + log.Errorf("VolumeProfileDelete retruned error for: %+v. Err: %v", obj, err) + return err + } + + // delete it from modeldb + err = obj.Delete() + if err != nil { + log.Errorf("Error deleting volumeProfile %s. Err: %v", obj.Key, err) + } + + // delete it from cache + delete(collections.volumeProfiles, key) + + return nil +} + +func (self *VolumeProfile) GetType() string { + return "volumeProfile" +} + +func (self *VolumeProfile) GetKey() string { + return self.Key +} + +func (self *VolumeProfile) Read() error { + if self.Key == "" { + log.Errorf("Empty key while trying to read volumeProfile object") + return errors.New("Empty key") + } + + return modeldb.ReadObj("volumeProfile", self.Key, self) +} + +func (self *VolumeProfile) Write() error { + if self.Key == "" { + log.Errorf("Empty key while trying to Write volumeProfile object") + return errors.New("Empty key") + } + + return modeldb.WriteObj("volumeProfile", self.Key, self) +} + +func (self *VolumeProfile) Delete() error { + if self.Key == "" { + log.Errorf("Empty key while trying to Delete volumeProfile object") + return errors.New("Empty key") + } + + return modeldb.DeleteObj("volumeProfile", self.Key) +} + +func restoreVolumeProfile() error { + strList, err := modeldb.ReadAllObj("volumeProfile") + if err != nil { + log.Errorf("Error reading volumeProfile list. Err: %v", err) + } + + for _, objStr := range strList { + // Parse the json model + var volumeProfile VolumeProfile + err = json.Unmarshal([]byte(objStr), &volumeProfile) + if err != nil { + log.Errorf("Error parsing object %s, Err %v", objStr, err) + return err + } + + // add it to the collection + collections.volumeProfiles[volumeProfile.Key] = &volumeProfile + } + + return nil +} + +// Validate a volumeProfile object +func ValidateVolumeProfile(obj *VolumeProfile) error { + // Validate key is correct + keyStr := obj.TenantName + ":" + obj.VolumeProfileName + if obj.Key != keyStr { + log.Errorf("Expecting VolumeProfile Key: %s. Got: %s", keyStr, obj.Key) + return errors.New("Invalid Key") + } + + // Validate each field + + return nil +} diff --git a/Godeps/_workspace/src/github.com/contiv/objmodel/contivModel/contivModel.js b/Godeps/_workspace/src/github.com/contiv/objmodel/contivModel/contivModel.js new file mode 100644 index 000000000..8dbcf9448 --- /dev/null +++ b/Godeps/_workspace/src/github.com/contiv/objmodel/contivModel/contivModel.js @@ -0,0 +1,4 @@ +// contivModel.js +// This file is auto generated by modelgen tool +// Do not edit this file manually + diff --git a/Godeps/_workspace/src/github.com/contiv/objmodel/contivModel/endpointGroup.json b/Godeps/_workspace/src/github.com/contiv/objmodel/contivModel/endpointGroup.json new file mode 100644 index 000000000..0c9910937 --- /dev/null +++ b/Godeps/_workspace/src/github.com/contiv/objmodel/contivModel/endpointGroup.json @@ -0,0 +1,43 @@ +{ + "name": "contivModel", + "objects": [ + { + "name": "endpointGroup", + "type": "object", + "key": [ "tenantName", "groupName" ], + "properties": { + "groupName": { + "type": "string", + "description": "Endpoint group Name" + }, + "tenantName": { + "type": "string", + "description": "Tenant Name" + }, + "networkName": { + "type": "string" + }, + "policies": { + "type": "array", + "items": "string" + } + }, + "link-sets": { + "services": { + "ref": "service" + }, + "policies": { + "ref": "policy" + } + }, + "links": { + "tenant": { + "ref": "tenant" + }, + "network": { + "ref": "network" + } + } + } + ] +} diff --git a/Godeps/_workspace/src/github.com/contiv/objmodel/contivModel/generate.sh b/Godeps/_workspace/src/github.com/contiv/objmodel/contivModel/generate.sh new file mode 100644 index 000000000..35de32c4a --- /dev/null +++ b/Godeps/_workspace/src/github.com/contiv/objmodel/contivModel/generate.sh @@ -0,0 +1 @@ +$GOPATH/bin/modelgen -s ./ -o ./ diff --git a/Godeps/_workspace/src/github.com/contiv/objmodel/contivModel/network.json b/Godeps/_workspace/src/github.com/contiv/objmodel/contivModel/network.json new file mode 100644 index 000000000..30a7b2888 --- /dev/null +++ b/Godeps/_workspace/src/github.com/contiv/objmodel/contivModel/network.json @@ -0,0 +1,53 @@ +{ + "name": "contivModel", + "objects": [ + { + "name": "network", + "type": "object", + "key": [ "tenantName", "networkName"], + "properties": { + "networkName": { + "type": "string", + "title": "Network name", + "length": 64 + }, + "tenantName": { + "type": "string", + "title": "Tenant Name", + "length": 64 + }, + "isPublic": { + "type": "bool" + }, + "isPrivate": { + "type": "bool" + }, + "encap": { + "type": "string", + "format": "^(vlan|vxlan)$" + }, + "subnet": { + "type": "string", + "format": "^([0-9]{1,3}?.[0-9]{1,3}?.[0-9]{1,3}?.[0-9]{1,3}?/[0-9]{1,2}?)$" + }, + "defaultGw": { + "type": "string", + "format": "^([0-9]{1,3}?.[0-9]{1,3}?.[0-9]{1,3}?.[0-9]{1,3}?)$" + } + }, + "link-sets": { + "services": { + "ref": "service" + }, + "endpointGroups": { + "ref": "endpointGroup" + } + }, + "links": { + "tenant": { + "ref": "tenant" + } + } + } + ] +} diff --git a/Godeps/_workspace/src/github.com/contiv/objmodel/contivModel/policy.json b/Godeps/_workspace/src/github.com/contiv/objmodel/contivModel/policy.json new file mode 100644 index 000000000..788f5952e --- /dev/null +++ b/Godeps/_workspace/src/github.com/contiv/objmodel/contivModel/policy.json @@ -0,0 +1,34 @@ +{ + "name": "contivModel", + "objects": [ + { + "name": "policy", + "type": "object", + "key": [ "tenantName", "policyName" ], + "properties": { + "policyName": { + "type": "string", + "description": "Policy Name" + }, + "tenantName": { + "type": "string", + "description": "Tenant Name" + } + }, + "link-sets": { + "endpointGroups": { + "ref": "endpointGroup" + }, + "rules": { + "ref": "rule" + + } + }, + "links": { + "tenant": { + "ref": "tenant" + } + } + } + ] +} diff --git a/Godeps/_workspace/src/github.com/contiv/objmodel/contivModel/rule.json b/Godeps/_workspace/src/github.com/contiv/objmodel/contivModel/rule.json new file mode 100644 index 000000000..517447bc4 --- /dev/null +++ b/Godeps/_workspace/src/github.com/contiv/objmodel/contivModel/rule.json @@ -0,0 +1,55 @@ +{ + "name": "contivModel", + "objects": [ + { + "name": "rule", + "type": "object", + "key": [ "tenantName", "policyName", "ruleName" ], + "properties": { + "ruleName": { + "type": "string", + "title": "Rule Name", + "length": 64 + }, + "policyName": { + "type": "string", + "title": "Policy Name", + "length": 64 + }, + "tenantName": { + "type": "string", + "title": "Tenant Name", + "length": 64 + }, + "direction": { + "type": "string", + "format": "^(in|out)$" + }, + "endpointGroup": { + "type": "string", + "length": 64 + }, + "network": { + "type": "string", + "length": 64 + }, + "ipAddress": { + "type": "string" + }, + "protocol": { + "type": "string", + "format": "^(tcp|udp|icmp|[0-9]{1,3}?)$" + }, + "port": { + "type": "int", + "max": 65535 + } + }, + "link-sets": { + "policies": { + "ref": "policy" + } + } + } + ] +} diff --git a/Godeps/_workspace/src/github.com/contiv/objmodel/contivModel/service.json b/Godeps/_workspace/src/github.com/contiv/objmodel/contivModel/service.json new file mode 100644 index 000000000..a58816766 --- /dev/null +++ b/Godeps/_workspace/src/github.com/contiv/objmodel/contivModel/service.json @@ -0,0 +1,73 @@ +{ + "name": "contivModel", + "objects": [ + { + "name": "service", + "type": "object", + "key": [ "tenantName", "appName", "serviceName" ], + "properties": { + "serviceName": { + "type": "string", + "description": "Service Name" + }, + "appName": { + "type": "string", + "description": "Application Name" + }, + "tenantName": { + "type": "string", + "description": "Tenant Name" + }, + "imageName": { + "type": "string" + }, + "cpu": { + "type": "string" + }, + "memory": { + "type": "string" + }, + "command": { + "type": "string" + }, + "environment": { + "type": "array", + "items": "string" + }, + "scale": { + "type": "int" + }, + "endpointGroups": { + "type": "array", + "items": "string" + }, + "networks": { + "type": "array", + "items": "string" + }, + "volumeProfile": { + "type": "string" + } + }, + "link-sets": { + "networks": { + "ref": "network" + }, + "endpointGroups": { + "ref": "endpointGroup" + }, + "instances": { + "ref": "serviceInstance" + } + }, + "links": { + "app": { + "ref": "app" + }, + "volumeProfile": { + "ref": "volumeProfile" + } + } + } + ] +} diff --git a/Godeps/_workspace/src/github.com/contiv/objmodel/contivModel/serviceInstance.json b/Godeps/_workspace/src/github.com/contiv/objmodel/contivModel/serviceInstance.json new file mode 100644 index 000000000..45f062ef6 --- /dev/null +++ b/Godeps/_workspace/src/github.com/contiv/objmodel/contivModel/serviceInstance.json @@ -0,0 +1,40 @@ +{ + "name": "contivModel", + "objects": [ + { + "name": "serviceInstance", + "type": "object", + "key": [ "tenantName", "appName", "serviceName", "instanceId" ], + "properties": { + "instanceId": { + "type": "string", + "description": "Service instance id" + }, + "tenantName": { + "type": "string", + "description": "Tenant Name" + }, + "appName": { + "type": "string" + }, + "serviceName": { + "type": "string" + }, + "volumes": { + "type": "array", + "items": "string" + } + }, + "link-sets": { + "volumes": { + "ref": "volume" + } + }, + "links": { + "service": { + "ref": "service" + } + } + } + ] +} diff --git a/Godeps/_workspace/src/github.com/contiv/objmodel/contivModel/tenant.json b/Godeps/_workspace/src/github.com/contiv/objmodel/contivModel/tenant.json new file mode 100644 index 000000000..c37a7896e --- /dev/null +++ b/Godeps/_workspace/src/github.com/contiv/objmodel/contivModel/tenant.json @@ -0,0 +1,54 @@ +{ + "name": "contivModel", + "objects": [ + { + "name": "tenant", + "type": "object", + "key": [ "tenantName" ], + "properties": { + "tenantName": { + "type": "string", + "title": "Tenant Name", + "length": 64 + }, + "subnetPool": { + "type": "string", + "format": "^([0-9]{1,3}?.[0-9]{1,3}?.[0-9]{1,3}?.[0-9]{1,3}?/[0-9]{1,2}?)$" + }, + "subnetLen": { + "type": "int", + "min": 1, + "max": 32 + }, + "vlans": { + "type": "string", + "format": "^([0-9]{1,4}?-[0-9]{1,4}?)$" + }, + "vxlans": { + "type": "string", + "format": "^([0-9]{1,8}?-[0-9]{1,8}?)$" + } + }, + "link-sets": { + "networks": { + "ref": "network" + }, + "apps": { + "ref": "app" + }, + "endpointGroups": { + "ref": "endpointGroup" + }, + "policies": { + "ref": "policy" + }, + "volumes": { + "ref": "volume" + }, + "volumeProfiles": { + "ref": "volumeProfile" + } + } + } + ] +} diff --git a/Godeps/_workspace/src/github.com/contiv/objmodel/contivModel/volume.json b/Godeps/_workspace/src/github.com/contiv/objmodel/contivModel/volume.json new file mode 100644 index 000000000..748e5da85 --- /dev/null +++ b/Godeps/_workspace/src/github.com/contiv/objmodel/contivModel/volume.json @@ -0,0 +1,42 @@ +{ + "name": "contivModel", + "objects": [ + { + "name": "volume", + "type": "object", + "key": [ "tenantName", "volumeName" ], + "properties": { + "volumeName": { + "type": "string", + "description": "Volume Name" + }, + "tenantName": { + "type": "string", + "description": "Tenant Name" + }, + "datastoreType": { + "type": "string" + }, + "poolName": { + "type": "string" + }, + "size": { + "type": "string" + }, + "mountPoint": { + "type": "string" + } + }, + "link-sets": { + "serviceInstances": { + "ref": "serviceInstance" + } + }, + "links": { + "tenant": { + "ref": "tenant" + } + } + } + ] +} diff --git a/Godeps/_workspace/src/github.com/contiv/objmodel/contivModel/volumeProfile.json b/Godeps/_workspace/src/github.com/contiv/objmodel/contivModel/volumeProfile.json new file mode 100644 index 000000000..56b4cff26 --- /dev/null +++ b/Godeps/_workspace/src/github.com/contiv/objmodel/contivModel/volumeProfile.json @@ -0,0 +1,42 @@ +{ + "name": "contivModel", + "objects": [ + { + "name": "volumeProfile", + "type": "object", + "key": [ "tenantName", "volumeProfileName" ], + "properties": { + "volumeProfileName": { + "type": "string", + "description": "Volume profile Name" + }, + "tenantName": { + "type": "string", + "description": "Tenant Name" + }, + "datastoreType": { + "type": "string" + }, + "poolName": { + "type": "string" + }, + "size": { + "type": "string" + }, + "mountPoint": { + "type": "string" + } + }, + "link-sets": { + "services": { + "ref": "service" + } + }, + "links": { + "tenant": { + "ref": "tenant" + } + } + } + ] +} diff --git a/Godeps/_workspace/src/github.com/contiv/objmodel/objdb/modeldb/modeldb.go b/Godeps/_workspace/src/github.com/contiv/objmodel/objdb/modeldb/modeldb.go new file mode 100644 index 000000000..8f08634c9 --- /dev/null +++ b/Godeps/_workspace/src/github.com/contiv/objmodel/objdb/modeldb/modeldb.go @@ -0,0 +1,116 @@ +/*** +Copyright 2014 Cisco Systems Inc. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package modeldb + +// Wrapper for persistently storing object model + +import ( + "github.com/contiv/objmodel/objdb/objdbClient" + + log "github.com/Sirupsen/logrus" +) + +type ModelObj interface { + GetType() string + GetKey() string + Read() error + Write() error +} + +// Link is a one way relattion between two objects +type Link struct { + ObjType string `json:"type,omitempty"` + ObjKey string `json:"key,omitempty"` +} + +// AddLink adds a one way link to target object +func AddLink(link *Link, obj ModelObj) { + link.ObjType = obj.GetType() + link.ObjKey = obj.GetKey() +} + +// RemoveLink removes a one way link +func RemoveLink(link *Link, obj ModelObj) { + link.ObjType = "" + link.ObjKey = "" +} + +// Add a link into linkset. initialize the linkset if required +func AddLinkSet(linkSet *(map[string]Link), obj ModelObj) error { + // Allocate the linkset if its nil + if *linkSet == nil { + *linkSet = make(map[string]Link) + } + + // add the link to map + (*linkSet)[obj.GetKey()] = Link{ + ObjType: obj.GetType(), + ObjKey: obj.GetKey(), + } + + return nil +} + +func RemoveLinkSet(linkSet *(map[string]Link), obj ModelObj) error { + // check is linkset is nil + if *linkSet == nil { + return nil + } + + // remove the link from map + delete(*linkSet, obj.GetKey()) + + return nil +} + +// persistent database +var cdb = objdbClient.NewClient() + +func WriteObj(objType, objKey string, value interface{}) error { + key := "/modeldb/" + objType + "/" + objKey + err := cdb.SetObj(key, value) + if err != nil { + log.Errorf("Error storing object %s. Err: %v", key, err) + return err + } + + return nil +} + +func ReadObj(objType, objKey string, retVal interface{}) error { + key := "/modeldb/" + objType + "/" + objKey + err := cdb.GetObj(key, retVal) + if err != nil { + log.Errorf("Error reading object: %s. Err: %v", key, err) + } + + return nil +} + +func DeleteObj(objType, objKey string) error { + key := "/modeldb/" + objType + "/" + objKey + err := cdb.DelObj(key) + if err != nil { + log.Errorf("Error deleting object: %s. Err: %v", key, err) + } + + return nil +} + +func ReadAllObj(objType string) ([]string, error) { + key := "/modeldb/" + objType + "/" + return cdb.ListDir(key) +} diff --git a/Godeps/_workspace/src/github.com/contiv/objmodel/objdb/objdbApi.go b/Godeps/_workspace/src/github.com/contiv/objmodel/objdb/objdbApi.go new file mode 100644 index 000000000..5ecbf6b2d --- /dev/null +++ b/Godeps/_workspace/src/github.com/contiv/objmodel/objdb/objdbApi.go @@ -0,0 +1,134 @@ +package objdb + +import ( + "sync" + + log "github.com/Sirupsen/logrus" +) + +// Lock event types +const ( + LockAcquired = iota // Successfully acquired + LockReleased // explicitly released + LockAcquireTimeout // Timeout trying to acquire lock + LockAcquireError // Error while acquiring + LockRefreshError // Error during ttl refresh + LockLost // We lost the lock +) + +// Lock Event notifications +type LockEvent struct { + EventType uint // Type of event +} + +// Lock interface +type LockInterface interface { + // Acquire a lock. + // Give up acquiring lock after timeout seconds. if timeout is 0, wait forever + Acquire(timeout uint64) error + + // Release the lock. This explicitly releases the lock and cleans up all state + // If we were still waiting to acquire lock, it stops it and cleans up + Release() error + + // For debug purposes only. + // Just stops refreshing the lock + Kill() error + + // Get the event channel + EventChan() <-chan LockEvent + + // Is the lock acquired + IsAcquired() bool + + // Get current holder of the lock + GetHolder() string +} + +// Information about a service +// Notes: +// There could be multiple instances of a service. hostname:port uniquely +// identify an instance of a service +type ServiceInfo struct { + ServiceName string // Name of the service + HostAddr string // Host name or IP address where its running + Port int // Port number where its listening +} + +const ( + WatchServiceEventAdd = iota // New Service endpoint added + WatchServiceEventDel // A service endpoint was deleted + WatchServiceEventError // Error occurred while watching for service +) + +type WatchServiceEvent struct { + EventType uint // event type + ServiceInfo ServiceInfo // Information about the service +} + +// Plugin API +type ObjdbApi interface { + // Initialize the plugin, only called once + Init(seedHosts []string) error + + // Return local address used by conf store + GetLocalAddr() (string, error) + + // Get a Key from conf store + GetObj(key string, retValue interface{}) error + + // Set a key in conf store + SetObj(key string, value interface{}) error + + // Remove an object + DelObj(key string) error + + // List all objects in a directory + ListDir(key string) ([]string, error) + + // Create a new lock + NewLock(name string, holderId string, ttl uint64) (LockInterface, error) + + // Register a service + // Service is registered with a ttl for 60sec and a goroutine is created + // to refresh the ttl. + RegisterService(serviceInfo ServiceInfo) error + + // List all end points for a service + GetService(name string) ([]ServiceInfo, error) + + // Watch for addition/deletion of service end points + WatchService(name string, eventCh chan WatchServiceEvent, stopCh chan bool) error + + // Deregister a service + // This removes the service from the registry and stops the refresh groutine + DeregisterService(serviceInfo ServiceInfo) error +} + +var ( + // List of plugins available + pluginList = make(map[string]ObjdbApi) + pluginMutex = new(sync.Mutex) +) + +// Register a plugin +func RegisterPlugin(name string, plugin ObjdbApi) error { + pluginMutex.Lock() + defer pluginMutex.Unlock() + pluginList[name] = plugin + + return nil +} + +// Return the plugin by name +func GetPlugin(name string) ObjdbApi { + // Find the conf store + pluginMutex.Lock() + defer pluginMutex.Unlock() + if pluginList[name] == nil { + log.Errorf("Confstore Plugin %s not registered", name) + log.Fatal("Confstore plugin not registered") + } + + return pluginList[name] +} diff --git a/Godeps/_workspace/src/github.com/contiv/objmodel/objdb/objdbClient/objdbClient.go b/Godeps/_workspace/src/github.com/contiv/objmodel/objdb/objdbClient/objdbClient.go new file mode 100644 index 000000000..8653060ca --- /dev/null +++ b/Godeps/_workspace/src/github.com/contiv/objmodel/objdb/objdbClient/objdbClient.go @@ -0,0 +1,26 @@ +package objdbClient + +import ( + log "github.com/Sirupsen/logrus" + "github.com/contiv/objmodel/objdb" + "github.com/contiv/objmodel/objdb/plugins" +) + +// Create a new conf store +func NewClient() objdb.ObjdbApi { + defaultConfStore := "etcd" + + // Init all plugins + plugins.Init() + + // Get the plugin + plugin := objdb.GetPlugin(defaultConfStore) + + // Initialize the objdb client + if err := plugin.Init([]string{}); err != nil { + log.Errorf("Error initializing confstore plugin. Err: %v", err) + log.Fatal("Error initializing confstore plugin") + } + + return plugin +} diff --git a/Godeps/_workspace/src/github.com/contiv/objmodel/objdb/objdbClient/objdbClient_test.go b/Godeps/_workspace/src/github.com/contiv/objmodel/objdb/objdbClient/objdbClient_test.go new file mode 100644 index 000000000..aab609302 --- /dev/null +++ b/Godeps/_workspace/src/github.com/contiv/objmodel/objdb/objdbClient/objdbClient_test.go @@ -0,0 +1,310 @@ +package objdbClient + +import ( + "fmt" + "os" + "runtime" + "testing" + "time" + + "github.com/contiv/objmodel/objdb" + + log "github.com/Sirupsen/logrus" +) + +type JsonObj struct { + Value string +} + +// New objdb client +var client = NewClient() + +func TestMain(m *testing.M) { + runtime.GOMAXPROCS(runtime.NumCPU()) + os.Exit(m.Run()) +} + +// Perform Set/Get operation on default conf store +func TestSetGet(t *testing.T) { + // Set + setVal := JsonObj{ + Value: "test1", + } + + if err := client.SetObj("/contiv.io/test", setVal); err != nil { + fmt.Printf("Fatal setting key. Err: %v\n", err) + t.Fatalf("Fatal setting key") + } + + var retVal JsonObj + + if err := client.GetObj("/contiv.io/test", &retVal); err != nil { + fmt.Printf("Fatal getting key. Err: %v\n", err) + t.Fatalf("Fatal getting key") + } + + if retVal.Value != "test1" { + fmt.Printf("Got invalid response: %+v\n", retVal) + t.Fatalf("Got invalid response") + } + + if err := client.DelObj("/contiv.io/test"); err != nil { + t.Fatalf("Fatal deleting test object. Err: %v", err) + } + + fmt.Printf("Set/Get/Del test successful\n") +} + +func TestLockAcquireRelease(t *testing.T) { + // Create a lock + lock1, err := client.NewLock("master", "hostname1", 10) + if err != nil { + t.Fatal(err) + } + + lock2, err := client.NewLock("master", "hostname2", 10) + if err != nil { + t.Fatal(err) + } + + // Acquire the master lock + if err := lock1.Acquire(0); err != nil { + t.Fatalf("Fatal acquiring lock1") + } + + time.Sleep(100 * time.Millisecond) + + // Try to acquire the same lock again. This should fail + if err := lock2.Acquire(0); err != nil { + t.Fatalf("Fatal acquiring lock2") + } + + cnt := 1 + for { + select { + case event := <-lock1.EventChan(): + fmt.Printf("Event on Lock1: %+v\n\n", event) + if event.EventType == objdb.LockAcquired { + fmt.Printf("Master lock acquired by Lock1\n") + } + case event := <-lock2.EventChan(): + fmt.Printf("Event on Lock2: %+v\n\n", event) + if event.EventType == objdb.LockAcquired { + fmt.Printf("Master lock acquired by Lock2\n") + } + case <-time.After(100 * time.Millisecond): + if cnt == 1 { + fmt.Printf("100 ms timer. releasing Lock1\n\n") + // At this point, lock1 should be holding the lock + if !lock1.IsAcquired() { + t.Fatalf("Lock1 failed to acquire lock\n\n") + } + + // Release lock1 so that lock2 can acquire it + lock1.Release() + cnt++ + } else { + fmt.Printf("200 ms timer. checking if lock2 is acquired\n\n") + + // At this point, lock2 should be holding the lock + if !lock2.IsAcquired() { + t.Fatalf("Lock2 failed to acquire lock\n\n") + } + + fmt.Printf("Success. Lock2 Successfully acquired. releasing it\n") + // we are done with the test + lock2.Release() + + return + } + } + } +} + +func TestLockAcquireTimeout(t *testing.T) { + fmt.Printf("\n\n\n =========================================================== \n\n\n") + // Create a lock + lock1, err := client.NewLock("master", "hostname1", 10) + if err != nil { + t.Fatal(err) + } + + lock2, err := client.NewLock("master", "hostname2", 10) + if err != nil { + t.Fatal(err) + } + + // Acquire the lock + + if err := lock1.Acquire(0); err != nil { + t.Fatalf("Fatal acquiring lock1") + } + + time.Sleep(100 * time.Millisecond) + + if err := lock2.Acquire(2); err != nil { + t.Fatalf("Fatal acquiring lock2") + } + + for { + select { + case event := <-lock1.EventChan(): + fmt.Printf("Event on Lock1: %+v\n\n", event) + if event.EventType == objdb.LockAcquired { + fmt.Printf("Master lock acquired by Lock1\n") + } + case event := <-lock2.EventChan(): + fmt.Printf("Event on Lock2: %+v\n\n", event) + if event.EventType != objdb.LockAcquireTimeout { + fmt.Printf("Invalid event on Lock2\n") + } else { + fmt.Printf("Lock2 timeout as expected") + } + case <-time.After(1 * time.Millisecond): + fmt.Printf("1sec timer. releasing Lock1\n\n") + // At this point, lock1 should be holding the lock + if !lock1.IsAcquired() { + t.Fatalf("Lock1 failed to acquire lock\n\n") + } + lock1.Release() + + return + } + } +} + +func TestServiceRegister(t *testing.T) { + // Service info + service1Info := objdb.ServiceInfo{ + ServiceName: "athena", + HostAddr: "10.10.10.10", + Port: 4567, + } + service2Info := objdb.ServiceInfo{ + ServiceName: "athena", + HostAddr: "10.10.10.10", + Port: 4568, + } + + // register it + if err := client.RegisterService(service1Info); err != nil { + t.Fatalf("Fatal registering service. Err: %+v\n", err) + } + log.Infof("Registered service: %+v", service1Info) + + if err := client.RegisterService(service2Info); err != nil { + t.Fatalf("Fatal registering service. Err: %+v\n", err) + } + log.Infof("Registered service: %+v", service2Info) + + resp, err := client.GetService("athena") + if err != nil { + t.Fatalf("Fatal getting service. Err: %+v\n", err) + } + + log.Infof("Got service list: %+v\n", resp) + + if (len(resp) < 2) || (resp[0] != service1Info) || (resp[1] != service2Info) { + t.Fatalf("Resp service list did not match input") + } + + // Wait a while to make sure background refresh is working correctly + time.Sleep(5 * time.Millisecond) + + resp, err = client.GetService("athena") + if err != nil { + t.Fatalf("Fatal getting service. Err: %+v\n", err) + } + + log.Infof("Got service list: %+v\n", resp) + + if (len(resp) < 2) || (resp[0] != service1Info) || (resp[1] != service2Info) { + t.Fatalf("Resp service list did not match input") + } +} + +func TestServiceDeregister(t *testing.T) { + // Service info + service1Info := objdb.ServiceInfo{ + ServiceName: "athena", + HostAddr: "10.10.10.10", + Port: 4567, + } + service2Info := objdb.ServiceInfo{ + ServiceName: "athena", + HostAddr: "10.10.10.10", + Port: 4568, + } + + // register it + if err := client.DeregisterService(service1Info); err != nil { + t.Fatalf("Fatal deregistering service. Err: %+v\n", err) + } + + if err := client.DeregisterService(service2Info); err != nil { + t.Fatalf("Fatal deregistering service. Err: %+v\n", err) + } + + time.Sleep(time.Millisecond * 1) +} + +func TestServiceWatch(t *testing.T) { + service1Info := objdb.ServiceInfo{ + ServiceName: "athena", + HostAddr: "10.10.10.10", + Port: 4567, + } + + // register it + + if err := client.RegisterService(service1Info); err != nil { + t.Fatalf("Fatal registering service. Err: %+v\n", err) + } + log.Infof("Registered service: %+v", service1Info) + + // Create event channel + eventChan := make(chan objdb.WatchServiceEvent, 1) + stopChan := make(chan bool, 1) + + // Start watching for service + + if err := client.WatchService("athena", eventChan, stopChan); err != nil { + t.Fatalf("Fatal watching service. Err %v", err) + } + + cnt := 1 + for { + select { + case srvEvent := <-eventChan: + log.Infof("\n----\nReceived event: %+v\n----", srvEvent) + case <-time.After(time.Millisecond * time.Duration(10)): + service2Info := objdb.ServiceInfo{ + ServiceName: "athena", + HostAddr: "10.10.10.11", + Port: 4567, + } + if cnt == 1 { + // register it + if err := client.RegisterService(service2Info); err != nil { + t.Fatalf("Fatal registering service. Err: %+v\n", err) + } + log.Infof("Registered service: %+v", service2Info) + } else if cnt == 5 { + // deregister it + if err := client.DeregisterService(service2Info); err != nil { + t.Fatalf("Fatal deregistering service. Err: %+v\n", err) + } + log.Infof("Deregistered service: %+v", service2Info) + } else if cnt == 7 { + // Stop the watch + stopChan <- true + + // wait a little and exit + time.Sleep(time.Millisecond) + + return + } + cnt++ + } + } +} diff --git a/Godeps/_workspace/src/github.com/contiv/objmodel/objdb/plugins/etcdClient/etcdClient.go b/Godeps/_workspace/src/github.com/contiv/objmodel/objdb/plugins/etcdClient/etcdClient.go new file mode 100644 index 000000000..82d312a9d --- /dev/null +++ b/Godeps/_workspace/src/github.com/contiv/objmodel/objdb/plugins/etcdClient/etcdClient.go @@ -0,0 +1,211 @@ +package etcdClient + +import ( + "encoding/json" + "errors" + "io/ioutil" + "net/http" + "strings" + "sync" + + "github.com/contiv/objmodel/objdb" + + log "github.com/Sirupsen/logrus" + "github.com/contiv/go-etcd/etcd" +) + +type EtcdPlugin struct { + client *etcd.Client // etcd client + + serviceDb map[string]*serviceState + mutex *sync.Mutex +} + +type selfData struct { + Name string `json:"name"` +} + +type member struct { + Name string `json:"name"` + ClientURLs []string `json:"clientURLs"` +} + +type memData struct { + Members []member `json:"members"` +} + +// etcd plugin state +var etcdPlugin = &EtcdPlugin{mutex: new(sync.Mutex)} + +// Register the plugin +func InitPlugin() { + objdb.RegisterPlugin("etcd", etcdPlugin) +} + +// Initialize the etcd client +func (self *EtcdPlugin) Init(machines []string) error { + self.mutex.Lock() + defer self.mutex.Unlock() + // Create a new client + self.client = etcd.NewClient(machines) + if self.client == nil { + log.Fatal("Error creating etcd client.") + return errors.New("Error creating etcd client") + } + + // Set strong consistency + self.client.SetConsistency(etcd.STRONG_CONSISTENCY) + + // Initialize service DB + self.serviceDb = make(map[string]*serviceState) + + return nil +} + +// Get an object +func (self *EtcdPlugin) GetObj(key string, retVal interface{}) error { + keyName := "/contiv.io/obj/" + key + + // Get the object from etcd client + resp, err := self.client.Get(keyName, false, false) + if err != nil { + log.Errorf("Error getting key %s. Err: %v", keyName, err) + return err + } + + // Parse JSON response + if err := json.Unmarshal([]byte(resp.Node.Value), retVal); err != nil { + log.Errorf("Error parsing object %s, Err %v", resp.Node.Value, err) + return err + } + + return nil +} + +// Recursive function to look thru each directory and get the files +func recursAddNode(node *etcd.Node, list []string) []string { + for _, innerNode := range node.Nodes { + // add only the files. + if !innerNode.Dir { + list = append(list, innerNode.Value) + } else { + list = recursAddNode(innerNode, list) + } + } + + return list +} + +// Get a list of objects in a directory +func (self *EtcdPlugin) ListDir(key string) ([]string, error) { + keyName := "/contiv.io/obj/" + key + + // Get the object from etcd client + resp, err := self.client.Get(keyName, true, true) + if err != nil { + return nil, nil + } + + if !resp.Node.Dir { + log.Errorf("ListDir response is not a directory") + return nil, errors.New("Response is not directory") + } + + retList := make([]string, 0) + // Call a recursive function to recurse thru each directory and get all files + // Warning: assumes directory itself is not interesting to the caller + // Warning2: there is also an assumption that keynames are not required + // Which means, caller has to derive the key from value :( + retList = recursAddNode(resp.Node, retList) + + return retList, nil +} + +// Save an object, create if it doesnt exist +func (self *EtcdPlugin) SetObj(key string, value interface{}) error { + keyName := "/contiv.io/obj/" + key + + // JSON format the object + jsonVal, err := json.Marshal(value) + if err != nil { + log.Errorf("Json conversion error. Err %v", err) + return err + } + + // Set it via etcd client + if _, err := self.client.Set(keyName, string(jsonVal[:]), 0); err != nil { + log.Errorf("Error setting key %s, Err: %v", keyName, err) + return err + } + + return nil +} + +// Remove an object +func (self *EtcdPlugin) DelObj(key string) error { + keyName := "/contiv.io/obj/" + key + + // Remove it via etcd client + if _, err := self.client.Delete(keyName, false); err != nil { + log.Errorf("Error removing key %s, Err: %v", keyName, err) + return err + } + + return nil +} + +// Get JSON output from a http request +func httpGetJson(url string, data interface{}) (interface{}, error) { + res, err := http.Get(url) + if err != nil { + log.Errorf("Error during http get. Err: %v", err) + return nil, err + } + body, err := ioutil.ReadAll(res.Body) + if err != nil { + log.Errorf("Error during ioutil readall. Err: %v", err) + return nil, err + } + + if err := json.Unmarshal(body, data); err != nil { + log.Errorf("Error during json unmarshall. Err: %v", err) + return nil, err + } + + log.Debugf("Results for (%s): %+v\n", url, data) + + return data, nil +} + +// Return the local address where etcd is listening +func (self *EtcdPlugin) GetLocalAddr() (string, error) { + var selfData selfData + // Get self state from etcd + if _, err := httpGetJson("http://localhost:2379/v2/stats/self", &selfData); err != nil { + log.Errorf("Error getting self state. Err: %v", err) + return "", errors.New("Error getting self state") + } + + var memData memData + + // Get member list from etcd + if _, err := httpGetJson("http://localhost:2379/v2/members", &memData); err != nil { + log.Errorf("Error getting self state. Err: %v", err) + return "", errors.New("Error getting self state") + } + + myName := selfData.Name + members := memData.Members + + for _, mem := range members { + if mem.Name == myName { + for _, clientUrl := range mem.ClientURLs { + hostStr := strings.TrimPrefix(clientUrl, "http://") + hostAddr := strings.Split(hostStr, ":")[0] + log.Infof("Got host addr: %s", hostAddr) + return hostAddr, nil + } + } + } + return "", errors.New("Address not found") +} diff --git a/Godeps/_workspace/src/github.com/contiv/objmodel/objdb/plugins/etcdClient/etcdLock.go b/Godeps/_workspace/src/github.com/contiv/objmodel/objdb/plugins/etcdClient/etcdLock.go new file mode 100644 index 000000000..018b404e8 --- /dev/null +++ b/Godeps/_workspace/src/github.com/contiv/objmodel/objdb/plugins/etcdClient/etcdLock.go @@ -0,0 +1,357 @@ +package etcdClient + +import ( + "sync" + "time" + + api "github.com/contiv/objmodel/objdb" + + log "github.com/Sirupsen/logrus" + "github.com/contiv/go-etcd/etcd" +) + +// Etcd error codes +const EtcdErrorCodeNotFound = 100 +const EtcdErrorCodeKeyExists = 105 + +// Lock object +type Lock struct { + name string + myId string + isAcquired bool + isReleased bool + holderId string + ttl uint64 + timeout uint64 + modifiedIndex uint64 + eventChan chan api.LockEvent + stopChan chan bool + watchCh chan *etcd.Response + watchStopCh chan bool + client *etcd.Client + mutex *sync.Mutex +} + +// Create a new lock +func (self *EtcdPlugin) NewLock(name string, myId string, ttl uint64) (api.LockInterface, error) { + // Create a lock + return &Lock{ + name: name, + myId: myId, + ttl: ttl, + client: self.client, + eventChan: make(chan api.LockEvent, 1), + stopChan: make(chan bool, 1), + watchCh: make(chan *etcd.Response, 1), + watchStopCh: make(chan bool, 1), + mutex: new(sync.Mutex), + }, nil +} + +// Acquire a lock +func (self *Lock) Acquire(timeout uint64) error { + self.mutex.Lock() + defer self.mutex.Unlock() + self.timeout = timeout + + // Acquire in background + go self.acquireLock() + + return nil +} + +// Release a lock +func (self *Lock) Release() error { + keyName := "/contiv.io/lock/" + self.name + + self.mutex.Lock() + defer self.mutex.Unlock() + + // Mark this as released + self.isReleased = true + + // Send stop signal on stop channel + self.stopChan <- true + + // If the lock was acquired, release it + if self.isAcquired { + // Update TTL on the lock + resp, err := self.client.CompareAndDelete(keyName, self.myId, self.modifiedIndex) + if err != nil { + log.Errorf("Error Deleting key. Err: %v", err) + } else { + log.Infof("Deleted key lock %s, Resp: %+v", keyName, resp) + + // Update modifiedIndex + self.modifiedIndex = resp.Node.ModifiedIndex + } + } + + return nil +} + +// Note: This is for debug/test purposes only +// Stop a lock without releasing it. +// Let the etcd TTL expiry release it +func (self *Lock) Kill() error { + self.mutex.Lock() + defer self.mutex.Unlock() + // Mark this as released + self.isReleased = true + + // Send stop signal on stop channel + self.stopChan <- true + + return nil +} + +// Return event channel +func (self *Lock) EventChan() <-chan api.LockEvent { + self.mutex.Lock() + defer self.mutex.Unlock() + return self.eventChan +} + +// Check if the lock is acquired +func (self *Lock) IsAcquired() bool { + self.mutex.Lock() + defer self.mutex.Unlock() + return self.isAcquired +} + +// Get current lock holder's Id +func (self *Lock) GetHolder() string { + self.mutex.Lock() + defer self.mutex.Unlock() + return self.holderId +} + +// *********************** Internal functions ************* +// Try acquiring a lock. +// This assumes its called in its own go routine +func (self *Lock) acquireLock() { + keyName := "/contiv.io/lock/" + self.name + + // Start a watch on the lock first so that we dont loose any notifications + go self.watchLock() + + // Wait in this loop forever till lock times out or released + for { + log.Infof("Getting the lock %s to see if its acquired", keyName) + // Get the key and see if we or someone else has already acquired the lock + resp, err := self.client.Get(keyName, false, false) + if err != nil { + if err.(*etcd.EtcdError).ErrorCode != EtcdErrorCodeNotFound { + log.Errorf("Error getting the key %s. Err: %v", keyName, err) + } else { + log.Infof("Lock %s does not exist. trying to acquire it", keyName) + } + + // Try to acquire the lock + resp, err := self.client.Create(keyName, self.myId, self.ttl) + if err != nil { + if err.(*etcd.EtcdError).ErrorCode != EtcdErrorCodeKeyExists { + log.Errorf("Error creating key %s. Err: %v", keyName, err) + } else { + log.Infof("Lock %s acquired by someone else", keyName) + } + } else { + log.Infof("Acquired lock %s. Resp: %#v, Node: %+v", keyName, resp, resp.Node) + + self.mutex.Lock() + // Successfully acquired the lock + self.isAcquired = true + self.holderId = self.myId + self.modifiedIndex = resp.Node.ModifiedIndex + self.mutex.Unlock() + + // Send acquired message to event channel + self.eventChan <- api.LockEvent{EventType: api.LockAcquired} + + // refresh it + self.refreshLock() + + self.mutex.Lock() + // If lock is released, we are done, else go back and try to acquire it + if self.isReleased { + self.mutex.Unlock() + return + } + self.mutex.Unlock() + } + } else if resp.Node.Value == self.myId { + log.Infof("Already Acquired key %s. Resp: %#v, Node: %+v", keyName, resp, resp.Node) + + self.mutex.Lock() + // We have already acquired the lock. just keep refreshing it + self.isAcquired = true + self.holderId = self.myId + self.modifiedIndex = resp.Node.ModifiedIndex + self.mutex.Unlock() + + // Send acquired message to event channel + self.eventChan <- api.LockEvent{EventType: api.LockAcquired} + + // Refresh lock + self.refreshLock() + + self.mutex.Lock() + // If lock is released, we are done, else go back and try to acquire it + if self.isReleased { + self.mutex.Unlock() + return + } + self.mutex.Unlock() + } else if resp.Node.Value != self.myId { + log.Infof("Lock already acquired by someone else. Resp: %+v, Node: %+v", resp, resp.Node) + + self.mutex.Lock() + // Set the current holder's Id + self.holderId = resp.Node.Value + self.mutex.Unlock() + + // Wait for changes on the lock + self.waitForLock() + + self.mutex.Lock() + if self.isReleased { + self.mutex.Unlock() + return + } + self.mutex.Unlock() + } + } +} + +// We couldnt acquire lock, Wait for changes on the lock +func (self *Lock) waitForLock() { + // If timeout is not specified, set it to high value + timeoutIntvl := time.Second * time.Duration(20000) + if self.timeout != 0 { + timeoutIntvl = time.Second * time.Duration(self.timeout) + } + + log.Infof("Waiting to acquire lock (%s/%s)", self.name, self.myId) + + // Create a timer + timer := time.NewTimer(timeoutIntvl) + defer timer.Stop() + + // Wait for changes + for { + // wait on watch channel for holder to release the lock + select { + case <-timer.C: + self.mutex.Lock() + if self.timeout != 0 { + self.mutex.Unlock() + log.Infof("Lock timeout on lock %s/%s", self.name, self.myId) + + self.eventChan <- api.LockEvent{EventType: api.LockAcquireTimeout} + + log.Infof("Lock acquire timed out. Stopping lock") + + self.watchStopCh <- true + + // Release the lock + self.Release() + + return + } + self.mutex.Unlock() + case watchResp := <-self.watchCh: + if watchResp != nil { + log.Debugf("Received watch notification(%s/%s): %+v", self.name, self.myId, watchResp) + + if watchResp.Action == "expire" || watchResp.Action == "delete" || + watchResp.Action == "compareAndDelete" { + log.Infof("Retrying to acquire lock") + return + } + } + case <-self.stopChan: + log.Infof("Stopping lock") + self.watchStopCh <- true + + return + } + } +} + +// Refresh lock +func (self *Lock) refreshLock() { + // Refresh interval is 40% of TTL + refreshIntvl := time.Second * time.Duration(self.ttl*3/10) + keyName := "/contiv.io/lock/" + self.name + + // Loop forever + for { + select { + case <-time.After(refreshIntvl): + // Update TTL on the lock + resp, err := self.client.CompareAndSwap(keyName, self.myId, self.ttl, + self.myId, self.modifiedIndex) + if err != nil { + log.Errorf("Error updating TTl. Err: %v", err) + + self.mutex.Lock() + // We are not master anymore + self.isAcquired = false + self.mutex.Unlock() + + // Send lock lost event + self.eventChan <- api.LockEvent{EventType: api.LockLost} + + // FIXME: trigger a lock lost event + return + } else { + log.Debugf("Refreshed TTL on lock %s, Resp: %+v", keyName, resp) + + self.mutex.Lock() + // Update modifiedIndex + self.modifiedIndex = resp.Node.ModifiedIndex + self.mutex.Unlock() + } + case watchResp := <-self.watchCh: + // Since we already acquired the lock, nothing to do here + // FIXME: see if we lost the lock + if watchResp != nil { + log.Debugf("Received watch notification for(%s/%s): %+v", + self.name, self.myId, watchResp) + } + case <-self.stopChan: + log.Infof("Stopping lock") + self.watchStopCh <- true + return + } + } +} + +// Watch for changes on the lock +func (self *Lock) watchLock() { + keyName := "/contiv.io/lock/" + self.name + + for { + resp, err := self.client.Watch(keyName, 0, false, self.watchCh, self.watchStopCh) + if err != nil { + if err != etcd.ErrWatchStoppedByUser { + log.Errorf("Error watching the key %s, Err %v", keyName, err) + } else { + log.Infof("Watch stopped for lock %s", keyName) + } + } else { + log.Infof("Got Watch Resp: %+v", resp) + } + + self.mutex.Lock() + // If the lock is released, we are done + if self.isReleased { + self.mutex.Unlock() + return + } + self.mutex.Unlock() + + // Wait for a second and go back to watching + time.Sleep(1 * time.Second) + } +} diff --git a/Godeps/_workspace/src/github.com/contiv/objmodel/objdb/plugins/etcdClient/etcdService.go b/Godeps/_workspace/src/github.com/contiv/objmodel/objdb/plugins/etcdClient/etcdService.go new file mode 100644 index 000000000..4bddf5572 --- /dev/null +++ b/Godeps/_workspace/src/github.com/contiv/objmodel/objdb/plugins/etcdClient/etcdService.go @@ -0,0 +1,224 @@ +package etcdClient + +import ( + "encoding/json" + "errors" + "strconv" + "strings" + "time" + + api "github.com/contiv/objmodel/objdb" + + log "github.com/Sirupsen/logrus" + "github.com/contiv/go-etcd/etcd" +) + +const SERVICE_TTL = 60 + +// Service state +type serviceState struct { + ServiceName string // Name of the service + HostAddr string // Host name or IP address where its running + Port int // Port number where its listening + + // Channel to stop ttl refresh + stopChan chan bool +} + +// Register a service +// Service is registered with a ttl for 60sec and a goroutine is created +// to refresh the ttl. +func (self *EtcdPlugin) RegisterService(serviceInfo api.ServiceInfo) error { + keyName := "/contiv.io/service/" + serviceInfo.ServiceName + "/" + + serviceInfo.HostAddr + ":" + strconv.Itoa(serviceInfo.Port) + + log.Infof("Registering service key: %s, value: %+v", keyName, serviceInfo) + + // JSON format the object + jsonVal, err := json.Marshal(serviceInfo) + if err != nil { + log.Errorf("Json conversion error. Err %v", err) + return err + } + + // Set it via etcd client + _, err = self.client.Set(keyName, string(jsonVal[:]), SERVICE_TTL) + if err != nil { + log.Errorf("Error setting key %s, Err: %v", keyName, err) + return err + } + + // Run refresh in background + stopChan := make(chan bool, 1) + go refreshService(self.client, keyName, string(jsonVal[:]), stopChan) + + // Store it in DB + self.serviceDb[keyName] = &serviceState{ + ServiceName: serviceInfo.ServiceName, + HostAddr: serviceInfo.HostAddr, + Port: serviceInfo.Port, + stopChan: stopChan, + } + + return nil +} + +// List all end points for a service +func (self *EtcdPlugin) GetService(name string) ([]api.ServiceInfo, error) { + keyName := "/contiv.io/service/" + name + "/" + + // Get the object from etcd client + resp, err := self.client.Get(keyName, true, true) + if err != nil { + log.Errorf("Error getting key %s. Err: %v", keyName, err) + return nil, err + } + + if !resp.Node.Dir { + log.Errorf("Err. Response is not a directory: %+v", resp.Node) + return nil, errors.New("Invalid Response from etcd") + } + + srvcList := make([]api.ServiceInfo, 0) + + // Parse each node in the directory + for _, node := range resp.Node.Nodes { + var respSrvc api.ServiceInfo + // Parse JSON response + err = json.Unmarshal([]byte(node.Value), &respSrvc) + if err != nil { + log.Errorf("Error parsing object %s, Err %v", node.Value, err) + return nil, err + } + + srvcList = append(srvcList, respSrvc) + } + + return srvcList, nil +} + +// Watch for a service +func (self *EtcdPlugin) WatchService(name string, + eventCh chan api.WatchServiceEvent, stopCh chan bool) error { + keyName := "/contiv.io/service/" + name + "/" + + // Create channels + watchCh := make(chan *etcd.Response, 1) + watchStopCh := make(chan bool, 1) + + // Start the watch thread + go func() { + log.Infof("Watching for service: %s", keyName) + // Start the watch + _, err := self.client.Watch(keyName, 0, true, watchCh, watchStopCh) + if (err != nil) && (err != etcd.ErrWatchStoppedByUser) { + log.Errorf("Error watching service %s. Err: %v", keyName, err) + + // Emit the event + eventCh <- api.WatchServiceEvent{EventType: api.WatchServiceEventError} + } + log.Infof("Watch thread exiting..") + }() + + // handle messages from watch service + go func() { + for { + select { + case watchResp := <-watchCh: + log.Debugf("Received event %#v\n Node: %#v", watchResp, watchResp.Node) + + // derive service info from key + srvKey := strings.TrimPrefix(watchResp.Node.Key, "/contiv.io/service/") + srvName := strings.Split(srvKey, "/")[0] + hostInfo := strings.Split(srvKey, "/")[1] + hostAddr := strings.Split(hostInfo, ":")[0] + portNum, _ := strconv.Atoi(strings.Split(hostInfo, ":")[1]) + + // Build service info + srvInfo := api.ServiceInfo{ + ServiceName: srvName, + HostAddr: hostAddr, + Port: portNum, + } + + // We ignore all events except Set/Delete/Expire + // Note that Set event doesnt exactly mean new service end point. + // If a service restarts and re-registers before it expired, we'll + // receive set again. receivers need to handle this case + if watchResp.Action == "set" { + log.Infof("Sending service add event: %+v", srvInfo) + // Send Add event + eventCh <- api.WatchServiceEvent{ + EventType: api.WatchServiceEventAdd, + ServiceInfo: srvInfo, + } + } else if (watchResp.Action == "delete") || + (watchResp.Action == "expire") { + + log.Infof("Sending service del event: %+v", srvInfo) + + // Send Delete event + eventCh <- api.WatchServiceEvent{ + EventType: api.WatchServiceEventDel, + ServiceInfo: srvInfo, + } + } + case stopReq := <-stopCh: + if stopReq { + // Stop watch and return + log.Infof("Stopping watch on %s", keyName) + watchStopCh <- true + return + } + } + } + }() + + return nil +} + +// Deregister a service +// This removes the service from the registry and stops the refresh groutine +func (self *EtcdPlugin) DeregisterService(serviceInfo api.ServiceInfo) error { + keyName := "/contiv.io/service/" + serviceInfo.ServiceName + "/" + + serviceInfo.HostAddr + ":" + strconv.Itoa(serviceInfo.Port) + + // Find it in the database + srvState := self.serviceDb[keyName] + if srvState == nil { + log.Errorf("Could not find the service in db %s", keyName) + return errors.New("Service not found") + } + + // stop the refresh thread and delete service + srvState.stopChan <- true + delete(self.serviceDb, keyName) + + // Delete the service instance + _, err := self.client.Delete(keyName, false) + if err != nil { + log.Errorf("Error getting key %s. Err: %v", keyName, err) + return err + } + + return nil +} + +// Keep refreshing the service every 30sec +func refreshService(client *etcd.Client, keyName string, keyVal string, stopChan chan bool) { + for { + select { + case <-time.After(time.Second * time.Duration(SERVICE_TTL/3)): + log.Debugf("Refreshing key: %s", keyName) + + _, err := client.Update(keyName, keyVal, SERVICE_TTL) + if err != nil { + log.Errorf("Error updating key %s, Err: %v", keyName, err) + } + + case <-stopChan: + log.Infof("Stop refreshing key: %s", keyName) + return + } + } +} diff --git a/Godeps/_workspace/src/github.com/contiv/objmodel/objdb/plugins/plugins.go b/Godeps/_workspace/src/github.com/contiv/objmodel/objdb/plugins/plugins.go new file mode 100644 index 000000000..436371d19 --- /dev/null +++ b/Godeps/_workspace/src/github.com/contiv/objmodel/objdb/plugins/plugins.go @@ -0,0 +1,10 @@ +package plugins + +import ( + "github.com/contiv/objmodel/objdb/plugins/etcdClient" +) + +func Init() { + // Initialize all conf store plugins + etcdClient.InitPlugin() +} diff --git a/drivers/ovsSwitch.go b/drivers/ovsSwitch.go index 93f9addfc..efea6ce2e 100644 --- a/drivers/ovsSwitch.go +++ b/drivers/ovsSwitch.go @@ -25,8 +25,11 @@ import ( "github.com/contiv/ofnet" log "github.com/Sirupsen/logrus" + "github.com/vishvananda/netlink" ) +const useVethPair = true + // OvsSwitch represents on OVS bridge instance type OvsSwitch struct { bridgeName string @@ -159,11 +162,67 @@ func (sw *OvsSwitch) DeleteNetwork(pktTag uint16, extPktTag uint32) error { return nil } +// createVethPair creates veth interface pairs with specified name +func createVethPair(name1, name2 string) error { + log.Infof("Creating Veth pairs with name: %s, %s", name1, name2) + + // Veth pair params + veth := &netlink.Veth{ + LinkAttrs: netlink.LinkAttrs{ + Name: name1, + TxQLen: 0, + }, + PeerName: name2, + } + + // Create the veth pair + if err := netlink.LinkAdd(veth); err != nil { + log.Errorf("error creating veth pair: %v", err) + return err + } + + return nil +} + +// setLinkUp sets the link up +func setLinkUp(name string) error { + iface, err := netlink.LinkByName(name) + if err != nil { + return err + } + return netlink.LinkSetUp(iface) +} + // CreatePort creates a port in ovs switch -func (sw *OvsSwitch) CreatePort(intfName, intfType string, cfgEp *OvsCfgEndpointState, - pktTag int) error { - // Ask OVSDB driver to add/delete the port - err := sw.ovsdbDriver.CreatePort(intfName, intfType, cfgEp.ID, pktTag) +func (sw *OvsSwitch) CreatePort(intfName string, cfgEp *OvsCfgEndpointState, pktTag int) error { + var ovsPortName string + var ovsIntfType string + if useVethPair { + // Generate interface + ovsPortName = strings.Replace(intfName, "port", "vport", 1) + ovsIntfType = "" + + // Create a Veth pair + err := createVethPair(intfName, ovsPortName) + if err != nil { + log.Errorf("Error creating veth pairs. Err: %v", err) + return err + } + + // Set the OVS side of the port as up + err = setLinkUp(ovsPortName) + if err != nil { + log.Errorf("Error setting link %s up. Err: %v", ovsPortName, err) + return err + } + } else { + ovsPortName = intfName + ovsIntfType = "internal" + + } + + // Ask OVSDB driver to add the port + err := sw.ovsdbDriver.CreatePort(ovsPortName, ovsIntfType, cfgEp.ID, pktTag) if err != nil { return err } @@ -176,10 +235,6 @@ func (sw *OvsSwitch) CreatePort(intfName, intfType string, cfgEp *OvsCfgEndpoint // Wait a little for OVS to create the interface time.Sleep(300 * time.Millisecond) - if intfType != "internal" { - log.Fatalf("Not expecting interface type :%s.", intfType) - } - // Set the interface mac address err = netutils.SetInterfaceMac(intfName, cfgEp.MacAddress) if err != nil { @@ -190,9 +245,9 @@ func (sw *OvsSwitch) CreatePort(intfName, intfType string, cfgEp *OvsCfgEndpoint // Add the endpoint to ofnet if sw.netType == "vxlan" { // Get the openflow port number for the interface - ofpPort, err := sw.ovsdbDriver.GetOfpPortNo(intfName) + ofpPort, err := sw.ovsdbDriver.GetOfpPortNo(ovsPortName) if err != nil { - log.Errorf("Could not find the OVS port %s. Err: %v", intfName, err) + log.Errorf("Could not find the OVS port %s. Err: %v", ovsPortName, err) return err } @@ -209,7 +264,7 @@ func (sw *OvsSwitch) CreatePort(intfName, intfType string, cfgEp *OvsCfgEndpoint // Add the local port to ofnet err = sw.ofnetAgent.AddLocalEndpoint(endpoint) if err != nil { - log.Errorf("Error adding local port %s to ofnet. Err: %v", intfName, err) + log.Errorf("Error adding local port %s to ofnet. Err: %v", ovsPortName, err) return err } } @@ -219,12 +274,20 @@ func (sw *OvsSwitch) CreatePort(intfName, intfType string, cfgEp *OvsCfgEndpoint // UpdatePort updates an OVS port without creating it func (sw *OvsSwitch) UpdatePort(intfName string, cfgEp *OvsCfgEndpointState, pktTag int) error { + var ovsPortName string + if useVethPair { + // Generate interface + ovsPortName = strings.Replace(intfName, "port", "vport", 1) + } else { + ovsPortName = intfName + } + // Add the endpoint to ofnet if sw.netType == "vxlan" { // Get the openflow port number for the interface - ofpPort, err := sw.ovsdbDriver.GetOfpPortNo(intfName) + ofpPort, err := sw.ovsdbDriver.GetOfpPortNo(ovsPortName) if err != nil { - log.Errorf("Could not find the OVS port %s. Err: %v", intfName, err) + log.Errorf("Could not find the OVS port %s. Err: %v", ovsPortName, err) return err } @@ -241,7 +304,7 @@ func (sw *OvsSwitch) UpdatePort(intfName string, cfgEp *OvsCfgEndpointState, pkt // Add the local port to ofnet err = sw.ofnetAgent.AddLocalEndpoint(endpoint) if err != nil { - log.Errorf("Error adding local port %s to ofnet. Err: %v", intfName, err) + log.Errorf("Error adding local port %s to ofnet. Err: %v", ovsPortName, err) return err } } @@ -251,6 +314,7 @@ func (sw *OvsSwitch) UpdatePort(intfName string, cfgEp *OvsCfgEndpointState, pkt // DeletePort removes a port from OVS func (sw *OvsSwitch) DeletePort(epOper *OvsOperEndpointState) error { + if epOper.VtepIP != "" { return nil } @@ -260,23 +324,31 @@ func (sw *OvsSwitch) DeletePort(epOper *OvsOperEndpointState) error { return err } + var ovsPortName string + if useVethPair { + // Generate interface + ovsPortName = strings.Replace(intfName, "port", "vport", 1) + } else { + ovsPortName = intfName + } + // Remove info from ofnet if sw.netType == "vxlan" { // Get the openflow port number for the interface - ofpPort, err := sw.ovsdbDriver.GetOfpPortNo(intfName) + ofpPort, err := sw.ovsdbDriver.GetOfpPortNo(ovsPortName) if err != nil { - log.Errorf("Could not find the OVS port %s. Err: %v", intfName, err) + log.Errorf("Could not find the OVS port %s. Err: %v", ovsPortName, err) return err } err = sw.ofnetAgent.RemoveLocalEndpoint(ofpPort) if err != nil { - log.Errorf("Error removing port %s from ofnet. Err: %v", intfName, err) + log.Errorf("Error removing port %s from ofnet. Err: %v", ovsPortName, err) } } // Delete it from ovsdb - err = sw.ovsdbDriver.DeletePort(intfName) + err = sw.ovsdbDriver.DeletePort(ovsPortName) if err != nil { return err } diff --git a/drivers/ovsdriver.go b/drivers/ovsdriver.go index 044e03c91..0ced4fb0a 100644 --- a/drivers/ovsdriver.go +++ b/drivers/ovsdriver.go @@ -17,11 +17,12 @@ package drivers import ( "encoding/json" + "errors" "fmt" - "github.com/contiv/netplugin/core" - log "github.com/Sirupsen/logrus" + "github.com/contiv/netplugin/core" + "github.com/contiv/netplugin/mgmtfn/dockplugin/libnetClient" ) type oper int @@ -153,6 +154,31 @@ func (d *OvsDriver) Deinit() { } } +const driverName = "netplugin" + +// createDockNet Creates a network in docker daemon +func createDockNet(networkName string) error { + api := libnetClient.NewRemoteAPI("") + + // Check if the network already exists + nw, err := api.NetworkByName(networkName) + if err == nil && nw.Type() == driverName { + return nil + } else if err == nil && nw.Type() != driverName { + log.Errorf("Network name %s used by another driver %s", networkName, nw.Type()) + return errors.New("Network name used by another driver") + } + + // Create network + _, err = api.NewNetwork(driverName, networkName) + if err != nil { + log.Errorf("Error creating network %s. Err: %v", networkName, err) + return err + } + + return nil +} + // CreateNetwork creates a network by named identifier func (d *OvsDriver) CreateNetwork(id string) error { cfgNw := OvsCfgNetworkState{} @@ -164,6 +190,13 @@ func (d *OvsDriver) CreateNetwork(id string) error { } log.Infof("create net %s \n", cfgNw.ID) + // Create the network in docker + err = createDockNet(cfgNw.ID) + if err != nil { + log.Errorf("Error creating network %s in docker. Err: %v", cfgNw.ID, err) + return err + } + // Find the switch based on network type var sw *OvsSwitch if cfgNw.PktTagType == "vxlan" { @@ -203,7 +236,6 @@ func (d *OvsDriver) CreateEndpoint(id string) error { var ( err error intfName string - intfType string ) cfgEp := &OvsCfgEndpointState{} @@ -273,10 +305,9 @@ func (d *OvsDriver) CreateEndpoint(id string) error { return err } intfName = d.getIntfName() - intfType = "internal" // Ask the switch to create the port - err = sw.CreatePort(intfName, intfType, cfgEp, cfgNw.PktTag) + err = sw.CreatePort(intfName, cfgEp, cfgNw.PktTag) if err != nil { log.Errorf("Error creating port %s. Err: %v", intfName, err) return err diff --git a/mgmtfn/dockplugin/dockplugin.go b/mgmtfn/dockplugin/dockplugin.go new file mode 100644 index 000000000..4e16a2170 --- /dev/null +++ b/mgmtfn/dockplugin/dockplugin.go @@ -0,0 +1,541 @@ +/*** +Copyright 2014 Cisco Systems Inc. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package dockplugin + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "io/ioutil" + "net" + "net/http" + "os" + "path" + "strings" + "time" + + log "github.com/Sirupsen/logrus" + "github.com/contiv/netplugin/drivers" + "github.com/contiv/netplugin/mgmtfn/dockplugin/libnetClient" + "github.com/contiv/netplugin/netmaster/client" + "github.com/contiv/netplugin/netmaster/intent" + "github.com/docker/docker/pkg/plugins" + "github.com/docker/libnetwork/drivers/remote/api" + "github.com/gorilla/mux" +) + +const pluginPath = "/run/docker/plugins" +const driverName = "netplugin" + +// InitDockPlugin initializes the docker plugin +func InitDockPlugin() error { + hostname, err := os.Hostname() + if err != nil { + log.Fatalf("Could not retrieve hostname: %v", err) + } + + var ( + tenantName = "default" + ) + + log.Debugf("Configuring router") + + router := mux.NewRouter() + s := router.Headers("Accept", "application/vnd.docker.plugins.v1+json"). + Methods("POST").Subrouter() + + dispatchMap := map[string]func(http.ResponseWriter, *http.Request){ + "/Plugin.Activate": activate(hostname), + "/Plugin.Deactivate": deactivate(hostname), + "/NetworkDriver.CreateNetwork": createNetwork(), + "/NetworkDriver.DeleteNetwork": deleteNetwork(), + "/NetworkDriver.CreateEndpoint": createEndpoint(tenantName, hostname), + "/NetworkDriver.DeleteEndpoint": deleteEndpoint(tenantName, hostname), + "/NetworkDriver.EndpointOperInfo": endpointInfo, + "/NetworkDriver.Join": join(), + "/NetworkDriver.Leave": leave(), + } + + for dispatchPath, dispatchFunc := range dispatchMap { + s.HandleFunc(dispatchPath, logHandler(dispatchPath, dispatchFunc)) + } + + driverPath := path.Join(pluginPath, driverName) + ".sock" + os.Remove(driverPath) + os.MkdirAll(pluginPath, 0700) + + go func() { + l, err := net.ListenUnix("unix", &net.UnixAddr{Name: driverPath, Net: "unix"}) + if err != nil { + panic(err) + } + + log.Infof("docker plugin listening on %s", driverPath) + http.Serve(l, router) + l.Close() + log.Infof("docker plugin closing %s", driverPath) + }() + + return nil +} + +func logHandler(name string, actionFunc func(http.ResponseWriter, *http.Request)) func(http.ResponseWriter, *http.Request) { + return func(w http.ResponseWriter, r *http.Request) { + + // Debug logs + buf := new(bytes.Buffer) + io.Copy(buf, r.Body) + log.Debugf("Dispatching %s with %v", name, strings.TrimSpace(string(buf.Bytes()))) + var writer *io.PipeWriter + r.Body, writer = io.Pipe() + go func() { + io.Copy(writer, buf) + writer.Close() + }() + + // Perform the action + actionFunc(w, r) + } +} + +func httpError(w http.ResponseWriter, message string, err error) { + fullError := fmt.Sprintf("%s %v", message, err) + + content, errc := json.Marshal(api.Response{Err: fullError}) + if errc != nil { + log.Warnf("Error received marshalling error response: %v, original error: %s", errc, fullError) + return + } + + log.Warnf("Returning HTTP error handling plugin negotiation: %s", fullError) + http.Error(w, string(content), http.StatusInternalServerError) +} + +func logEvent(typ string) { + log.Infof("Handling %q event", typ) +} + +// activate the plugin and register it as a network driver. +func deactivate(hostname string) func(http.ResponseWriter, *http.Request) { + return func(w http.ResponseWriter, r *http.Request) { + logEvent("deactivate") + } +} + +// activate the plugin and register it as a network driver. +func activate(hostname string) func(http.ResponseWriter, *http.Request) { + return func(w http.ResponseWriter, r *http.Request) { + logEvent("activate") + + content, err := json.Marshal(plugins.Manifest{Implements: []string{"NetworkDriver"}}) + if err != nil { + httpError(w, "Could not generate bootstrap response", err) + return + } + + w.Write(content) + } +} + +func deleteNetwork() func(http.ResponseWriter, *http.Request) { + return func(w http.ResponseWriter, r *http.Request) { + logEvent("delete network") + + content, err := ioutil.ReadAll(r.Body) + if err != nil { + httpError(w, "Could not read delete network request", err) + return + } + + dnreq := api.DeleteNetworkRequest{} + if err := json.Unmarshal(content, &dnreq); err != nil { + httpError(w, "Could not read delete network request", err) + return + } + + dnresp := api.DeleteNetworkResponse{} + content, err = json.Marshal(dnresp) + if err != nil { + httpError(w, "Could not generate delete network response", err) + return + } + w.Write(content) + } +} + +func createNetwork() func(http.ResponseWriter, *http.Request) { + return func(w http.ResponseWriter, r *http.Request) { + logEvent("create network") + + content, err := ioutil.ReadAll(r.Body) + if err != nil { + httpError(w, "Could not read create network request", err) + return + } + + log.Infoln(string(content)) + + cnreq := api.CreateNetworkRequest{} + if err := json.Unmarshal(content, &cnreq); err != nil { + httpError(w, "Could not read create network request", err) + return + } + + log.Infof("CreateNetworkRequest: %+v", cnreq) + + cnresp := api.CreateNetworkResponse{} + content, err = json.Marshal(cnresp) + if err != nil { + httpError(w, "Could not generate create network response", err) + return + } + + w.Write(content) + } +} + +func generateEndpoint(containerID, tenantName, networkName, hostname string) *intent.Config { + return &intent.Config{ + Tenants: []intent.ConfigTenant{ + { + Name: tenantName, + Networks: []intent.ConfigNetwork{ + { + Name: networkName, + Endpoints: []intent.ConfigEP{ + { + Container: containerID, + Host: hostname, + }, + }, + }, + }, + }, + }, + } +} + +func deleteEndpoint(tenantName, hostname string) func(http.ResponseWriter, *http.Request) { + return func(w http.ResponseWriter, r *http.Request) { + logEvent("delete endpoint") + + content, err := ioutil.ReadAll(r.Body) + if err != nil { + httpError(w, "Could not read delete endpoint request", err) + return + } + + der := api.DeleteEndpointRequest{} + if err := json.Unmarshal(content, &der); err != nil { + httpError(w, "Could not read delete endpoint request", err) + return + } + + networkName, err := GetNetworkName(der.NetworkID) + if err != nil { + log.Errorf("Error getting network name for UUID: %s. Err: %v", der.NetworkID, err) + httpError(w, "Could not get network name", err) + return + } + + if err := netdcliDel(generateEndpoint(der.EndpointID, tenantName, networkName, hostname)); err != nil { + httpError(w, "Could not create the endpoint", err) + return + } + + content, err = json.Marshal(api.DeleteEndpointResponse{}) + if err != nil { + httpError(w, "Could not generate delete endpoint response", err) + return + } + + w.Write(content) + } +} + +func createEndpoint(tenantName, hostname string) func(http.ResponseWriter, *http.Request) { + return func(w http.ResponseWriter, r *http.Request) { + // assumptions on options passed as of early v1: + // io.docker.network.endpoint.exposedports: docker's notion of exposed ports: + // * array of struct of Port, Proto (I presume 6 is ipv6 and 4 is ipv4) + // io.docker.network.endpoint.portmap: map of exposed ports to the host + // * structure follows: + // { + // "Proto": 6, + // "IP": "", + // "Port": 1234, + // "HostIP": "", + // "HostPort": 1234 + // } + // + + logEvent("create endpoint") + + content, err := ioutil.ReadAll(r.Body) + if err != nil { + httpError(w, "Could not read endpoint create request", err) + return + } + + cereq := api.CreateEndpointRequest{} + + if err := json.Unmarshal(content, &cereq); err != nil { + httpError(w, "Could not read endpoint create request", err) + return + } + + log.Infof("CreateEndpointRequest: %+v", cereq) + + networkName, err := GetNetworkName(cereq.NetworkID) + if err != nil { + log.Errorf("Error getting network name for UUID: %s. Err: %v", cereq.NetworkID, err) + httpError(w, "Could not get network name", err) + return + } + + // FIXME: + GetEndPointName(cereq.NetworkID, cereq.EndpointID) + + if err := netdcliAdd(generateEndpoint(cereq.EndpointID, tenantName, networkName, hostname)); err != nil { + httpError(w, "Could not create endpoint", err) + return + } + + time.Sleep(1 * time.Second) + + ep, err := netdcliGetEndpoint(networkName + "-" + cereq.EndpointID) + if err != nil { + httpError(w, "Could not find created endpoint", err) + return + } + + log.Debug(ep) + + nw, err := netdcliGetNetwork(networkName) + if err != nil { + httpError(w, "Could not find created endpoint", err) + return + } + + epResponse := api.CreateEndpointResponse{ + Interfaces: []*api.EndpointInterface{ + &api.EndpointInterface{ + ID: 1, + Address: fmt.Sprintf("%s/%d", ep[0].IPAddress, nw[0].SubnetLen), + }, + }, + } + + content, err = json.Marshal(epResponse) + if err != nil { + httpError(w, "Could not generate create endpoint response", err) + return + } + + w.Write(content) + } +} + +func endpointInfo(w http.ResponseWriter, r *http.Request) { + logEvent("endpoint info") + + content, err := ioutil.ReadAll(r.Body) + if err != nil { + httpError(w, "Could not read endpoint create request", err) + return + } + + epireq := api.EndpointInfoRequest{} + + if err := json.Unmarshal(content, &epireq); err != nil { + httpError(w, "Could not read endpoint create request", err) + return + } + + log.Infof("EndpointInfoRequest: %+v", epireq) + + resp, err := json.Marshal(api.EndpointInfoResponse{}) + if err != nil { + httpError(w, "Could not generate endpoint info response", err) + return + } + + w.Write(resp) +} + +func join() func(http.ResponseWriter, *http.Request) { + return func(w http.ResponseWriter, r *http.Request) { + logEvent("join") + + jr := api.JoinRequest{} + content, err := ioutil.ReadAll(r.Body) + if err != nil { + httpError(w, "Could not read join request", err) + return + } + + if err := json.Unmarshal(content, &jr); err != nil { + httpError(w, "Could not parse join request", err) + return + } + + log.Infof("JoinRequest: %+v", jr) + + networkName, err := GetNetworkName(jr.NetworkID) + if err != nil { + log.Errorf("Error getting network name for UUID: %s. Err: %v", jr.NetworkID, err) + httpError(w, "Could not get network name", err) + return + } + + // FIXME: + GetEndPointName(jr.NetworkID, jr.EndpointID) + + ep, err := netdcliGetEndpoint(networkName + "-" + jr.EndpointID) + if err != nil { + httpError(w, "Could not derive created interface", err) + return + } + + nw, err := netdcliGetNetwork(networkName) + if err != nil { + httpError(w, "Could not get network", err) + return + } + + // Inspect the container + /* + sbKey := strings.Split(jr.SandboxKey, "/") + cntId := sbKey[len(sbKey)-1] + log.Infof("Executing: docker inspect %s", cntId) + out, err := exec.Command("docker", "inspect", cntId).CombinedOutput() + log.Infof("docker ispect %s\nErr: %v\n%s", cntId, err, out) + + out, err := exec.Command("sh", "-c", "curl localhost:4243/v1.20/networks").CombinedOutput() + log.Infof("curl localhost:4243/v1.20/networks\nErr: %v\n%s", err, out) + */ + + content, err = json.Marshal(api.JoinResponse{ + InterfaceNames: []*api.InterfaceName{ + &api.InterfaceName{}, + &api.InterfaceName{ + SrcName: ep[0].PortName, + DstPrefix: "eth", + }, + }, + Gateway: nw[0].DefaultGw, + }) + + if err != nil { + httpError(w, "Could not generate join response", err) + return + } + + w.Write(content) + } +} + +func leave() func(http.ResponseWriter, *http.Request) { + return func(w http.ResponseWriter, r *http.Request) { + logEvent("leave") + w.WriteHeader(200) + } +} + +func netdcliAdd(payload *intent.Config) error { + c := client.New("localhost:9999") + log.Infof("netdcliAdd payload: %+v", payload) + if err := c.PostAddConfig(payload); err != nil { + println(err) + return err + } + + return nil +} + +func netdcliDel(payload *intent.Config) error { + c := client.New("localhost:9999") + return c.PostDeleteConfig(payload) +} + +func netdcliGetEndpoint(name string) ([]drivers.OvsOperEndpointState, error) { + c := client.New("localhost:9999") + content, err := c.GetEndpoint(name) + if err != nil { + return nil, err + } + + var endpoint []drivers.OvsOperEndpointState + + if err := json.Unmarshal(content, &endpoint); err != nil { + return nil, err + } + + return endpoint, nil +} + +func netdcliGetNetwork(name string) ([]drivers.OvsCfgNetworkState, error) { + var network []drivers.OvsCfgNetworkState + + c := client.New("localhost:9999") + content, err := c.GetNetwork(name) + if err != nil { + return network, err + } + + if err := json.Unmarshal(content, &network); err != nil { + return network, err + } + + return network, nil +} + +// GetNetworkName gets network name from network UUID +func GetNetworkName(nwID string) (string, error) { + api := libnetClient.NewRemoteAPI("") + + nw, err := api.NetworkByID(nwID) + if err != nil { + log.Infof("Error: %v", err) + return "", err + } + + log.Infof("Returning network name %s for ID %s", nw.Name(), nwID) + + return nw.Name(), nil +} + +// GetEndPointName Returns endpoint name from networkId, endpointId +func GetEndPointName(nwID, epID string) (string, error) { + api := libnetClient.NewRemoteAPI("") + + nw, err := api.NetworkByID(nwID) + if err != nil { + log.Infof("Error: %v", err) + return "", err + } + + ep, err := nw.EndpointByID(epID) + if err != nil { + log.Infof("Error: %v", err) + return "", err + } + + log.Infof("Returning endpoint name %s for ID %s/%s", ep.Name(), nwID, epID) + + return ep.Name(), nil +} diff --git a/mgmtfn/dockplugin/libnetClient/libnetClient.go b/mgmtfn/dockplugin/libnetClient/libnetClient.go new file mode 100644 index 000000000..a186882e1 --- /dev/null +++ b/mgmtfn/dockplugin/libnetClient/libnetClient.go @@ -0,0 +1,574 @@ +package libnetClient + +import ( + "bytes" + "crypto/tls" + "encoding/json" + "errors" + "fmt" + "io/ioutil" + "net" + "net/http" + "net/url" + "path" + "time" + + "github.com/docker/libnetwork" + "github.com/docker/libnetwork/config" + "github.com/docker/libnetwork/sandbox" + "github.com/docker/libnetwork/types" +) + +// networkResource is the body of the "get network" http response message +type networkResource struct { + Name string `json:"name"` + ID string `json:"id"` + Type string `json:"type"` + Endpoints []*endpointResource `json:"endpoints"` +} + +// endpointResource is the body of the "get endpoint" http response message +type endpointResource struct { + Name string `json:"name"` + ID string `json:"id"` + Network string `json:"network"` +} + +// containerResource is the body of "get service backend" response message +type containerResource struct { + ID string `json:"id"` + // will add more fields once labels change is in +} + +// endpointInfoResource is the body of the "get endpoint info" http response message +type endpointInfoResource struct { + Interfaces []interfaceResource `json:"interfaces"` + Gateway4 string `json:"gateway"` + Gateway6 string `json:"gateway6"` + Sandbox string `json:"sandbox"` +} + +type interfaceResource struct { + MAC string `json:"mac"` + Addr string `json:"addr"` + Addr6 string `json:"addr6"` +} + +/*********** + Body types + ************/ + +// networkCreate is the expected body of the "create network" http request message +type networkCreate struct { + Name string `json:"name"` + NetworkType string `json:"network_type"` + Options map[string]interface{} `json:"options"` +} + +// endpointCreate represents the body of the "create endpoint" http request message +type endpointCreate struct { + Name string `json:"name"` + ExposedPorts []types.TransportPort `json:"exposed_ports"` + PortMapping []types.PortBinding `json:"port_mapping"` +} + +// endpointJoin represents the expected body of the "join endpoint" or "leave endpoint" http request messages +type endpointJoin struct { + ContainerID string `json:"container_id"` + HostName string `json:"host_name"` + DomainName string `json:"domain_name"` + HostsPath string `json:"hosts_path"` + ResolvConfPath string `json:"resolv_conf_path"` + DNS []string `json:"dns"` + ExtraHosts []endpointExtraHost `json:"extra_hosts"` + ParentUpdates []endpointParentUpdate `json:"parent_updates"` + UseDefaultSandbox bool `json:"use_default_sandbox"` +} + +// servicePublish represents the body of the "publish service" http request message +type servicePublish struct { + Name string `json:"name"` + Network string `json:"network_name"` + ExposedPorts []types.TransportPort `json:"exposed_ports"` + PortMapping []types.PortBinding `json:"port_mapping"` +} + +// EndpointExtraHost represents the extra host object +type endpointExtraHost struct { + Name string `json:"name"` + Address string `json:"address"` +} + +// EndpointParentUpdate is the object carrying the information about the +// endpoint parent that needs to be updated +type endpointParentUpdate struct { + EndpointID string `json:"endpoint_id"` + Name string `json:"name"` + Address string `json:"address"` +} + +// ErrNotImplemented is returned for function that are not implemented +var ErrNotImplemented = errors.New("method not implemented") + +// Implements NetworkController by means of HTTP api +type remoteController struct { + baseURL *url.URL + client *http.Client +} + +// NewRemoteAPI returns an object implementing NetworkController +// interface that forwards the requests to the daemon +func NewRemoteAPI(daemonUrl string) libnetwork.NetworkController { + if daemonUrl == "" { + daemonUrl = "unix:///var/run/docker.sock" + } + url, err := url.Parse(daemonUrl) + if err != nil { + return nil + } + + if url.Scheme == "" || url.Scheme == "tcp" { + url.Scheme = "http" + } + return &remoteController{ + baseURL: url, + client: newHTTPClient(url, nil), + } +} + +// ConfigureNetworkDriver applies the passed options to the driver instance for the specified network type +func (c *remoteController) ConfigureNetworkDriver(networkType string, options map[string]interface{}) error { + return ErrNotImplemented +} + +// Config method returns the bootup configuration for the controller +func (c *remoteController) Config() config.Config { + panic("remoteController does not implement Config()") +} + +// Create a new network. The options parameter carries network specific options. +// Labels support will be added in the near future. +func (c *remoteController) NewNetwork(networkType, name string, options ...libnetwork.NetworkOption) (libnetwork.Network, error) { + url := "/networks" + + // TODO: handle options somehow + create := networkCreate{ + Name: name, + NetworkType: networkType, + } + + nid := "" + if err := c.httpPost(url, create, &nid); err != nil { + return nil, fmt.Errorf("http error: %v", err) + } + + return &remoteNetwork{ + rc: c, + nr: networkResource{ + Name: name, + ID: nid, + Type: networkType, + }, + }, nil +} + +// Networks returns the list of Network(s) managed by this controller. +func (c *remoteController) Networks() []libnetwork.Network { + nrs := []networkResource{} + if err := c.httpGet("/networks", &nrs); err != nil { + return nil + } + + ns := []libnetwork.Network{} + for _, nr := range nrs { + ns = append(ns, &remoteNetwork{c, nr}) + } + return ns +} + +// WalkNetworks uses the provided function to walk the Network(s) managed by this controller. +func (c *remoteController) WalkNetworks(walker libnetwork.NetworkWalker) { + for _, n := range c.Networks() { + if walker(n) { + return + } + } +} + +// NetworkByName returns the Network which has the passed name. If not found, the error ErrNoSuchNetwork is returned. +func (c *remoteController) NetworkByName(name string) (libnetwork.Network, error) { + ns := []networkResource{} + if err := c.httpGet("/networks?name="+name, &ns); err != nil { + return nil, err + } + + if len(ns) == 0 { + return nil, libnetwork.ErrNoSuchNetwork(name) + } + return &remoteNetwork{c, ns[0]}, nil +} + +// NetworkByID returns the Network which has the passed id. If not found, the error ErrNoSuchNetwork is returned. +func (c *remoteController) NetworkByID(id string) (libnetwork.Network, error) { + n := networkResource{} + if err := c.httpGet(path.Join("/networks", id), &n); err != nil { + if se, ok := err.(httpStatusErr); ok && se.Code() == http.StatusNotFound { + return nil, libnetwork.ErrNoSuchNetwork(id) + } + return nil, err + } + + return &remoteNetwork{c, n}, nil +} + +// GC triggers immediate garbage collection of resources which are garbage collected. +func (c *remoteController) GC() { + panic("remoteController does not implement GC()") +} + +// LeaveAll accepts a container id and attempts to leave all endpoints that the container has joined +func (c *remoteController) LeaveAll(id string) error { + return ErrNotImplemented +} + +type remoteNetwork struct { + rc *remoteController + nr networkResource +} + +// A user chosen name for this network. +func (n *remoteNetwork) Name() string { + return n.nr.Name +} + +// A system generated id for this network. +func (n *remoteNetwork) ID() string { + return n.nr.ID +} + +// The type of network, which corresponds to its managing driver. +func (n *remoteNetwork) Type() string { + return n.nr.Type +} + +// Create a new endpoint to this network symbolically identified by the +// specified unique name. The options parameter carry driver specific options. +// Labels support will be added in the near future. +func (n *remoteNetwork) CreateEndpoint(name string, options ...libnetwork.EndpointOption) (libnetwork.Endpoint, error) { + url := path.Join("/networks", n.nr.ID, "endpoints") + // TODO: process options somehow + create := endpointCreate{ + Name: name, + } + + eid := "" + if err := n.rc.httpPost(url, create, &eid); err != nil { + return nil, err + } + + return &remoteEndpoint{ + rc: n.rc, + networkID: n.nr.ID, + er: endpointResource{ + Name: name, + ID: eid, + Network: n.nr.Name, + }, + }, nil +} + +// Delete the network. +func (n *remoteNetwork) Delete() error { + url := path.Join("/networks", n.nr.ID) + return n.rc.httpDelete(url) +} + +// Endpoints returns the list of Endpoint(s) in this network. +func (n *remoteNetwork) Endpoints() []libnetwork.Endpoint { + endpoints := make([]libnetwork.Endpoint, 0, len(n.nr.Endpoints)) + for _, er := range n.nr.Endpoints { + endpoints = append(endpoints, &remoteEndpoint{ + rc: n.rc, + er: *er, + networkID: n.nr.ID, + }) + } + return endpoints +} + +// WalkEndpoints uses the provided function to walk the Endpoints +func (n *remoteNetwork) WalkEndpoints(walker libnetwork.EndpointWalker) { + for _, e := range n.Endpoints() { + if walker(e) { + return + } + } +} + +// EndpointByName returns the Endpoint which has the passed name. If not found, the error ErrNoSuchEndpoint is returned. +func (n *remoteNetwork) EndpointByName(name string) (libnetwork.Endpoint, error) { + // TODO: should this make an RPC + for _, er := range n.nr.Endpoints { + if er.Name == name { + return &remoteEndpoint{ + rc: n.rc, + er: *er, + networkID: n.nr.ID, + }, nil + } + } + return nil, libnetwork.ErrNoSuchEndpoint(name) +} + +// EndpointByID returns the Endpoint which has the passed id. If not found, the error ErrNoSuchEndpoint is returned. +func (n *remoteNetwork) EndpointByID(id string) (libnetwork.Endpoint, error) { + // TODO: should this make an RPC + for _, er := range n.nr.Endpoints { + if er.ID == id { + return &remoteEndpoint{ + rc: n.rc, + er: *er, + networkID: n.nr.ID, + }, nil + } + } + return nil, libnetwork.ErrNoSuchEndpoint(id) +} + +type remoteEndpoint struct { + rc *remoteController + networkID string + er endpointResource +} + +// A system generated id for this endpoint. +func (e *remoteEndpoint) ID() string { + return e.er.ID +} + +// Name returns the name of this endpoint. +func (e *remoteEndpoint) Name() string { + return e.er.Name +} + +// Network returns the name of the network to which this endpoint is attached. +func (e *remoteEndpoint) Network() string { + return e.er.Network +} + +// Join creates a new sandbox for the given container ID and populates the +// network resources allocated for the endpoint and joins the sandbox to +// the endpoint. It returns the sandbox key to the caller +func (e *remoteEndpoint) Join(containerID string, options ...libnetwork.EndpointOption) error { + url := path.Join("/networks", e.networkID, "endpoints", e.er.ID, "containers") + + //TODO: process options somehow + join := endpointJoin{ + ContainerID: containerID, + } + + sk := "" + return e.rc.httpPost(url, join, &sk) +} + +// Leave removes the sandbox associated with container ID and detaches +// the network resources populated in the sandbox +func (e *remoteEndpoint) Leave(containerID string, options ...libnetwork.EndpointOption) error { + url := path.Join("/networks", e.networkID, "endpoints", e.er.ID, "containers", containerID) + return e.rc.httpDelete(url) +} + +// Return certain operational data belonging to this endpoint +func (e *remoteEndpoint) Info() libnetwork.EndpointInfo { + url := path.Join("/networks", e.networkID, "endpoints", e.er.ID, "info") + eir := &endpointInfoResource{} + if err := e.rc.httpGet(url, eir); err != nil { + return nil + } + + return (*remoteEndpointInfo)(eir) +} + +// DriverInfo returns a collection of driver operational data related to this endpoint retrieved from the driver +func (e *remoteEndpoint) DriverInfo() (map[string]interface{}, error) { + return nil, ErrNotImplemented +} + +// ContainerInfo returns the info available at the endpoint about the attached container +func (e *remoteEndpoint) ContainerInfo() libnetwork.ContainerInfo { + return nil +} + +// Delete and detaches this endpoint from the network. +func (e *remoteEndpoint) Delete() error { + url := path.Join("/networks", e.networkID, "endpoints", e.er.ID) + return e.rc.httpDelete(url) +} + +// Retrieve the interfaces' statistics from the sandbox +func (e *remoteEndpoint) Statistics() (map[string]*sandbox.InterfaceStatistics, error) { + return nil, ErrNotImplemented +} + +type remoteEndpointInfo endpointInfoResource + +// InterfaceList returns an interface list which were assigned to the endpoint +// by the driver. This can be used after the endpoint has been created. +func (ei *remoteEndpointInfo) InterfaceList() []libnetwork.InterfaceInfo { + iis := []libnetwork.InterfaceInfo{} + for _, ii := range ei.Interfaces { + iis = append(iis, (*remoteInterfaceInfo)(&ii)) + } + return iis +} + +// Gateway returns the IPv4 gateway assigned by the driver. +// This will only return a valid value if a container has joined the endpoint. +func (ei *remoteEndpointInfo) Gateway() net.IP { + return net.ParseIP(ei.Gateway4) +} + +// GatewayIPv6 returns the IPv6 gateway assigned by the driver. +// This will only return a valid value if a container has joined the endpoint. +func (ei *remoteEndpointInfo) GatewayIPv6() net.IP { + return net.ParseIP(ei.Gateway6) +} + +// SandboxKey returns the sanbox key for the container which has joined +// the endpoint. If there is no container joined then this will return an +// empty string. +func (ei *remoteEndpointInfo) SandboxKey() string { + return ei.Sandbox +} + +type remoteInterfaceInfo interfaceResource + +// MacAddress returns the MAC address assigned to the endpoint. +func (ii *remoteInterfaceInfo) MacAddress() net.HardwareAddr { + if mac, err := net.ParseMAC(ii.MAC); err == nil { + return mac + } + return nil +} + +// Address returns the IPv4 address assigned to the endpoint. +func (ii *remoteInterfaceInfo) Address() net.IPNet { + ip, ipn, err := net.ParseCIDR(ii.Addr) + if err != nil || ip.To4() == nil { + return net.IPNet{} + } + ipn.IP = ip + return *ipn +} + +// AddressIPv6 returns the IPv6 address assigned to the endpoint. +func (ii *remoteInterfaceInfo) AddressIPv6() net.IPNet { + ip, ipn, err := net.ParseCIDR(ii.Addr) + if err != nil || ip.To16() == nil { + return net.IPNet{} + } + ipn.IP = ip + return *ipn +} + +const defaultTimeout = 30 * time.Second + +func newHTTPClient(u *url.URL, tlsConfig *tls.Config) *http.Client { + timeout := time.Duration(defaultTimeout) + httpTransport := &http.Transport{ + TLSClientConfig: tlsConfig, + } + + switch u.Scheme { + default: + httpTransport.Dial = func(proto, addr string) (net.Conn, error) { + return net.DialTimeout(proto, addr, timeout) + } + case "unix": + socketPath := u.Path + unixDial := func(proto, addr string) (net.Conn, error) { + return net.DialTimeout("unix", socketPath, timeout) + } + httpTransport.Dial = unixDial + // Override the main URL object so the HTTP lib won't complain + u.Scheme = "http" + u.Host = "unix.sock" + u.Path = "" + } + return &http.Client{Transport: httpTransport} +} + +type httpStatusErr int + +func (se httpStatusErr) Error() string { + return fmt.Sprintf("HTTP status error: %v", int(se)) +} + +func (se httpStatusErr) Code() int { + return int(se) +} + +func httpErr(resp *http.Response) error { + msg, _ := ioutil.ReadAll(resp.Body) + return fmt.Errorf("http status error: %v: %v", resp.StatusCode, string(msg)) +} + +func (rc *remoteController) httpGet(path string, res interface{}) error { + req, err := http.NewRequest("GET", rc.baseURL.String()+path, nil) + if err != nil { + return err + } + resp, err := rc.client.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + return httpErr(resp) + } + + return json.NewDecoder(resp.Body).Decode(res) +} + +func (rc *remoteController) httpPost(path string, body interface{}, res interface{}) error { + encoded, err := json.Marshal(body) + if err != nil { + return err + } + req, err := http.NewRequest("POST", rc.baseURL.String()+path, bytes.NewBuffer(encoded)) + if err != nil { + return err + } + req.Header.Set("Content-Type", "application/json") + resp, err := rc.client.Do(req) + // resp, err := rc.client.Post(url, "application/json", bytes.NewBuffer(encoded)) + if err != nil { + return err + } + defer resp.Body.Close() + + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + return httpErr(resp) + } + + return json.NewDecoder(resp.Body).Decode(res) +} + +func (rc *remoteController) httpDelete(path string) error { + req, err := http.NewRequest("DELETE", rc.baseURL.String()+path, nil) + if err != nil { + return err + } + resp, err := rc.client.Do(req) + if err != nil { + return err + } + resp.Body.Close() + + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + return httpErr(resp) + } + + return nil +} diff --git a/netmaster/main.go b/netmaster/main.go index 536fbf7f4..26a447b70 100644 --- a/netmaster/main.go +++ b/netmaster/main.go @@ -27,6 +27,7 @@ import ( "github.com/contiv/netplugin/drivers" "github.com/contiv/netplugin/netmaster/intent" "github.com/contiv/netplugin/netmaster/master" + "github.com/contiv/netplugin/netmaster/objApi" "github.com/contiv/netplugin/resources" "github.com/contiv/netplugin/state" "github.com/contiv/netplugin/utils" @@ -50,7 +51,8 @@ func usage() { } type daemon struct { - opts cliOpts + opts cliOpts + apiController *objApi.APIController } func initStateDriver(opts *cliOpts) (core.StateDriver, error) { @@ -143,6 +145,10 @@ func (d *daemon) execOpts() { func (d *daemon) ListenAndServe() { r := mux.NewRouter() + // Create a new api controller + d.apiController = objApi.NewAPIController(r) + + // Add REST routes s := r.Headers("Content-Type", "application/json").Methods("Post").Subrouter() s.HandleFunc(fmt.Sprintf("/%s", master.DesiredConfigRESTEndpoint), post(d.desiredConfig)) diff --git a/netmaster/master/netmaster.go b/netmaster/master/netmaster.go index 498162648..17de3783a 100644 --- a/netmaster/master/netmaster.go +++ b/netmaster/master/netmaster.go @@ -429,14 +429,13 @@ func validateNetworkConfig(tenant *intent.ConfigTenant) error { return err } -// CreateNetworks creates the necessary virtual networks for the tenant -// provided by ConfigTenant. -func CreateNetworks(stateDriver core.StateDriver, tenant *intent.ConfigTenant) error { +// CreateNetwork creates a network from intent +func CreateNetwork(network intent.ConfigNetwork, stateDriver core.StateDriver, tenantName string) error { var extPktTag, pktTag uint gCfg := gstate.Cfg{} gCfg.StateDriver = stateDriver - err := gCfg.Read(tenant.Name) + err := gCfg.Read(tenantName) if err != nil { log.Errorf("error reading tenant cfg state. Error: %s", err) return err @@ -448,108 +447,125 @@ func CreateNetworks(stateDriver core.StateDriver, tenant *intent.ConfigTenant) e } rm := core.ResourceManager(tempRm) - err = validateNetworkConfig(tenant) - if err != nil { - log.Errorf("error validating network config. Error: %s", err) - return err + // Create network state + nwCfg := &drivers.OvsCfgNetworkState{} + nwCfg.StateDriver = stateDriver + if nwCfg.Read(network.Name) == nil { + // TODO: check if parameters changed and apply an update if needed + return nil } - for _, network := range tenant.Networks { - nwCfg := &drivers.OvsCfgNetworkState{} - nwCfg.StateDriver = stateDriver - if nwCfg.Read(network.Name) == nil { - // TODO: check if parameters changed and apply an update if needed - continue - } - - // construct and update network state - nwMasterCfg := &NwConfig{} - nwMasterCfg.StateDriver = stateDriver - nwMasterCfg.Tenant = tenant.Name - nwMasterCfg.ID = network.Name - nwMasterCfg.PktTagType = network.PktTagType - nwMasterCfg.PktTag = network.PktTag - nwMasterCfg.SubnetIP, nwMasterCfg.SubnetLen, _ = netutils.ParseCIDR(network.SubnetCIDR) - nwMasterCfg.DefaultGw = network.DefaultGw - - nwCfg = &drivers.OvsCfgNetworkState{Tenant: nwMasterCfg.Tenant, - PktTagType: nwMasterCfg.PktTagType, - SubnetIP: nwMasterCfg.SubnetIP, SubnetLen: nwMasterCfg.SubnetLen} - nwCfg.StateDriver = stateDriver - nwCfg.ID = nwMasterCfg.ID + // construct and update network state + nwMasterCfg := &NwConfig{} + nwMasterCfg.StateDriver = stateDriver + nwMasterCfg.Tenant = tenantName + nwMasterCfg.ID = network.Name + nwMasterCfg.PktTagType = network.PktTagType + nwMasterCfg.PktTag = network.PktTag + nwMasterCfg.SubnetIP, nwMasterCfg.SubnetLen, _ = netutils.ParseCIDR(network.SubnetCIDR) + nwMasterCfg.DefaultGw = network.DefaultGw + + nwCfg = &drivers.OvsCfgNetworkState{ + Tenant: nwMasterCfg.Tenant, + PktTagType: nwMasterCfg.PktTagType, + SubnetIP: nwMasterCfg.SubnetIP, + SubnetLen: nwMasterCfg.SubnetLen, + } + nwCfg.StateDriver = stateDriver + nwCfg.ID = nwMasterCfg.ID - if nwMasterCfg.PktTagType == "" { - nwCfg.PktTagType = gCfg.Deploy.DefaultNetType - } - if nwMasterCfg.PktTag == "" { - if nwCfg.PktTagType == "vlan" { - pktTag, err = gCfg.AllocVLAN(rm) - if err != nil { - return err - } - } else if nwCfg.PktTagType == "vxlan" { - extPktTag, pktTag, err = gCfg.AllocVXLAN(rm) - if err != nil { - return err - } + if nwMasterCfg.PktTagType == "" { + nwCfg.PktTagType = gCfg.Deploy.DefaultNetType + } + if nwMasterCfg.PktTag == "" { + if nwCfg.PktTagType == "vlan" { + pktTag, err = gCfg.AllocVLAN(rm) + if err != nil { + return err } - - nwCfg.ExtPktTag = int(extPktTag) - nwCfg.PktTag = int(pktTag) - } else if nwMasterCfg.PktTagType == "vxlan" { - // XXX: take local vlan as config, instead of allocating it - // independently. Return erro for now, if user tries this config - return core.Errorf("Not handled. Need to introduce local-vlan config") - } else if nwMasterCfg.PktTagType == "vlan" { - nwCfg.PktTag, _ = strconv.Atoi(nwMasterCfg.PktTag) - // XXX: do configuration check, to make sure it is allowed - } - - if nwCfg.SubnetIP == "" { - nwCfg.SubnetLen = gCfg.Auto.AllocSubnetLen - nwCfg.SubnetIP, err = gCfg.AllocSubnet(rm) + } else if nwCfg.PktTagType == "vxlan" { + extPktTag, pktTag, err = gCfg.AllocVXLAN(rm) if err != nil { return err } } - nwCfg.DefaultGw = network.DefaultGw - if nwCfg.DefaultGw == "" { - // TBD: allocate per global policy - } + nwCfg.ExtPktTag = int(extPktTag) + nwCfg.PktTag = int(pktTag) + } else if nwMasterCfg.PktTagType == "vxlan" { + // XXX: take local vlan as config, instead of allocating it + // independently. Return erro for now, if user tries this config + return core.Errorf("Not handled. Need to introduce local-vlan config") + } else if nwMasterCfg.PktTagType == "vlan" { + nwCfg.PktTag, _ = strconv.Atoi(nwMasterCfg.PktTag) + // XXX: do configuration check, to make sure it is allowed + } - netutils.InitSubnetBitset(&nwCfg.IPAllocMap, nwCfg.SubnetLen) - err = nwCfg.Write() + if nwCfg.SubnetIP == "" { + nwCfg.SubnetLen = gCfg.Auto.AllocSubnetLen + nwCfg.SubnetIP, err = gCfg.AllocSubnet(rm) if err != nil { return err } + } - err = nwMasterCfg.Write() - if err != nil { - log.Errorf("error writing nw config. Error: %s", err) - return err - } + nwCfg.DefaultGw = network.DefaultGw + if nwCfg.DefaultGw == "" { + // TBD: allocate per global policy + } - if nwCfg.PktTagType == "vxlan" { + netutils.InitSubnetBitset(&nwCfg.IPAllocMap, nwCfg.SubnetLen) + err = nwCfg.Write() + if err != nil { + return err + } - readHost := &HostConfig{} - readHost.StateDriver = stateDriver - hostCfgs, err := readHost.ReadAll() - if err != nil { - if !strings.Contains(err.Error(), "Key not found") { - log.Errorf("error reading hosts during net add. Error: %s", err) - } + err = nwMasterCfg.Write() + if err != nil { + log.Errorf("error writing nw config. Error: %s", err) + return err + } + + if nwCfg.PktTagType == "vxlan" { + + readHost := &HostConfig{} + readHost.StateDriver = stateDriver + hostCfgs, err := readHost.ReadAll() + if err != nil { + if !strings.Contains(err.Error(), "Key not found") { + log.Errorf("error reading hosts during net add. Error: %s", err) } - for _, hostCfg := range hostCfgs { - host := hostCfg.(*HostConfig) - err = createVtep(stateDriver, host, nwCfg.ID) - if err != nil { - log.Errorf("error creating vtep. Error: %s", err) - } + } + for _, hostCfg := range hostCfgs { + host := hostCfg.(*HostConfig) + err = createVtep(stateDriver, host, nwCfg.ID) + if err != nil { + log.Errorf("error creating vtep. Error: %s", err) } } } + return nil +} + +// CreateNetworks creates the necessary virtual networks for the tenant +// provided by ConfigTenant. +func CreateNetworks(stateDriver core.StateDriver, tenant *intent.ConfigTenant) error { + // Validate the config + err := validateNetworkConfig(tenant) + if err != nil { + log.Errorf("error validating network config. Error: %s", err) + return err + } + + for _, network := range tenant.Networks { + err = CreateNetwork(network, stateDriver, tenant.Name) + if err != nil { + log.Errorf("Error creating network {%+v}. Err: %v", network, err) + return err + } + } + return err } diff --git a/netmaster/objApi/apiController.go b/netmaster/objApi/apiController.go new file mode 100644 index 000000000..9fed1ebd5 --- /dev/null +++ b/netmaster/objApi/apiController.go @@ -0,0 +1,671 @@ +/*** +Copyright 2014 Cisco Systems Inc. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package objApi + +import ( + "errors" + "fmt" + + "github.com/contiv/netplugin/netmaster/intent" + "github.com/contiv/netplugin/netmaster/master" + "github.com/contiv/netplugin/utils" + "github.com/contiv/objmodel/contivModel" + "github.com/contiv/objmodel/objdb/modeldb" + + log "github.com/Sirupsen/logrus" + "github.com/gorilla/mux" +) + +// APIController stores the api controller state +type APIController struct { + router *mux.Router +} + +var apiCtrler *APIController + +// NewAPIController creates a new controller +func NewAPIController(router *mux.Router) *APIController { + ctrler := new(APIController) + ctrler.router = router + + // initialize the model objects + contivModel.Init() + + // Register Callbacks + contivModel.RegisterAppCallbacks(ctrler) + contivModel.RegisterEndpointGroupCallbacks(ctrler) + contivModel.RegisterNetworkCallbacks(ctrler) + contivModel.RegisterPolicyCallbacks(ctrler) + contivModel.RegisterRuleCallbacks(ctrler) + contivModel.RegisterServiceCallbacks(ctrler) + contivModel.RegisterServiceInstanceCallbacks(ctrler) + contivModel.RegisterTenantCallbacks(ctrler) + contivModel.RegisterVolumeCallbacks(ctrler) + contivModel.RegisterVolumeProfileCallbacks(ctrler) + + // Register routes + contivModel.AddRoutes(router) + + // Add default tenant if it doesnt exist + tenant := contivModel.FindTenant("default") + if tenant == nil { + log.Infof("Creating default tenant") + err := contivModel.CreateTenant(&contivModel.Tenant{ + Key: "default", + TenantName: "default", + SubnetPool: "10.1.1.1/16", + SubnetLen: 24, + Vlans: "1-2000", + Vxlans: "10001-12000", + }) + if err != nil { + log.Fatalf("Error creating default tenant. Err: %v", err) + } + } + + return ctrler +} + +// AppCreate creates app state +func (ac *APIController) AppCreate(app *contivModel.App) error { + log.Infof("Received AppCreate: %+v", app) + + // Make sure tenant exists + if app.TenantName == "" { + return errors.New("Invalid tenant name") + } + + tenant := contivModel.FindTenant(app.TenantName) + if tenant == nil { + return errors.New("Tenant not found") + } + + // Setup links + modeldb.AddLink(&app.Links.Tenant, tenant) + modeldb.AddLinkSet(&tenant.LinkSets.Apps, app) + + // Save the tenant too since we added the links + err := tenant.Write() + if err != nil { + log.Errorf("Error updating tenant state(%+v). Err: %v", tenant, err) + return err + } + + return nil +} + +// AppUpdate updates app +func (ac *APIController) AppUpdate(app, params *contivModel.App) error { + log.Infof("Received AppUpdate: %+v, params: %+v", app, params) + + return nil +} + +// AppDelete delete the app +func (ac *APIController) AppDelete(app *contivModel.App) error { + log.Infof("Received AppDelete: %+v", app) + return nil +} + +// EndpointGroupCreate creates end point group +func (ac *APIController) EndpointGroupCreate(endpointGroup *contivModel.EndpointGroup) error { + log.Infof("Received EndpointGroupCreate: %+v", endpointGroup) + return nil +} + +// EndpointGroupUpdate updates endpoint group +func (ac *APIController) EndpointGroupUpdate(endpointGroup, params *contivModel.EndpointGroup) error { + log.Infof("Received EndpointGroupUpdate: %+v, params: %+v", endpointGroup, params) + return nil +} + +// EndpointGroupDelete deletes end point group +func (ac *APIController) EndpointGroupDelete(endpointGroup *contivModel.EndpointGroup) error { + log.Infof("Received EndpointGroupDelete: %+v", endpointGroup) + return nil +} + +// NetworkCreate creates network +func (ac *APIController) NetworkCreate(network *contivModel.Network) error { + log.Infof("Received NetworkCreate: %+v", network) + + // Make sure tenant exists + if network.TenantName == "" { + return errors.New("Invalid tenant name") + } + + tenant := contivModel.FindTenant(network.TenantName) + if tenant == nil { + return errors.New("Tenant not found") + } + + // Setup links + modeldb.AddLink(&network.Links.Tenant, tenant) + modeldb.AddLinkSet(&tenant.LinkSets.Networks, network) + + // Save the tenant too since we added the links + err := tenant.Write() + if err != nil { + log.Errorf("Error updating tenant state(%+v). Err: %v", tenant, err) + return err + } + + // Get the state driver + stateDriver, err := utils.GetStateDriver() + if err != nil { + return err + } + + // Build networ config + networkCfg := intent.ConfigNetwork{ + Name: network.NetworkName, + PktTagType: network.Encap, + PktTag: "", + SubnetCIDR: network.Subnet, + DefaultGw: network.DefaultGw, + } + + // Create the network + err = master.CreateNetwork(networkCfg, stateDriver, network.TenantName) + if err != nil { + log.Errorf("Error creating network {%+v}. Err: %v", network, err) + return err + } + + return nil +} + +// NetworkUpdate updates network +func (ac *APIController) NetworkUpdate(network, params *contivModel.Network) error { + log.Infof("Received NetworkUpdate: %+v, params: %+v", network, params) + return errors.New("Cant change network parameters after its created") +} + +// NetworkDelete deletes network +func (ac *APIController) NetworkDelete(network *contivModel.Network) error { + log.Infof("Received NetworkDelete: %+v", network) + + // Find the tenant + tenant := contivModel.FindTenant(network.TenantName) + if tenant == nil { + return errors.New("Tenant not found") + } + + // Remove link + modeldb.RemoveLinkSet(&tenant.LinkSets.Networks, network) + + // Save the tenant too since we added the links + err := tenant.Write() + if err != nil { + return err + } + + // Get the state driver + stateDriver, err := utils.GetStateDriver() + if err != nil { + return err + } + + // Delete the network + err = master.DeleteNetworkID(stateDriver, network.NetworkName) + if err != nil { + log.Errorf("Error deleting network %s. Err: %v", network.NetworkName, err) + } + + return nil +} + +// PolicyCreate creates policy +func (ac *APIController) PolicyCreate(policy *contivModel.Policy) error { + log.Infof("Received PolicyCreate: %+v", policy) + return nil +} + +// PolicyUpdate updates policy +func (ac *APIController) PolicyUpdate(policy, params *contivModel.Policy) error { + log.Infof("Received PolicyUpdate: %+v, params: %+v", policy, params) + return nil +} + +// PolicyDelete deletes policy +func (ac *APIController) PolicyDelete(policy *contivModel.Policy) error { + log.Infof("Received PolicyDelete: %+v", policy) + return nil +} + +// RuleCreate Creates the rule witin a policy +func (ac *APIController) RuleCreate(rule *contivModel.Rule) error { + log.Infof("Received RuleCreate: %+v", rule) + return nil +} + +// RuleUpdate updates the rule within a policy +func (ac *APIController) RuleUpdate(rule, params *contivModel.Rule) error { + log.Infof("Received RuleUpdate: %+v, params: %+v", rule, params) + return nil +} + +// RuleDelete deletes the rule within a policy +func (ac *APIController) RuleDelete(rule *contivModel.Rule) error { + log.Infof("Received RuleDelete: %+v", rule) + return nil +} + +// ServiceCreate creates service +func (ac *APIController) ServiceCreate(service *contivModel.Service) error { + log.Infof("Received ServiceCreate: %+v", service) + + // check params + if (service.TenantName == "") || (service.AppName == "") { + return errors.New("Invalid parameters") + } + + // Make sure tenant exists + tenant := contivModel.FindTenant(service.TenantName) + if tenant == nil { + return errors.New("Tenant not found") + } + + // Find the app this service belongs to + app := contivModel.FindApp(service.TenantName + ":" + service.AppName) + if app == nil { + return errors.New("App not found") + } + + // Setup links + modeldb.AddLink(&service.Links.App, app) + modeldb.AddLinkSet(&app.LinkSets.Services, service) + + // Save the app too since we added the links + err := app.Write() + if err != nil { + return err + } + + // Check if user specified any networks + if len(service.Networks) == 0 { + service.Networks = append(service.Networks, "privateNet") + } + + // link service with network + for _, netName := range service.Networks { + netKey := service.TenantName + ":" + netName + network := contivModel.FindNetwork(netKey) + if network == nil { + log.Errorf("Service: %s could not find network %s", service.Key, netKey) + return errors.New("Network not found") + } + + // Link the network + modeldb.AddLinkSet(&service.LinkSets.Networks, network) + modeldb.AddLinkSet(&network.LinkSets.Services, service) + + // save the network + err := network.Write() + if err != nil { + return err + } + } + + // Check if user specified any endpoint group for the service + if len(service.EndpointGroups) == 0 { + // Create one default endpointGroup per network + for _, netName := range service.Networks { + // params for default endpoint group + dfltEpgName := service.AppName + "." + service.ServiceName + "." + netName + endpointGroup := contivModel.EndpointGroup{ + Key: service.TenantName + ":" + dfltEpgName, + TenantName: service.TenantName, + NetworkName: netName, + GroupName: dfltEpgName, + } + + // Create default endpoint group for the service + err = contivModel.CreateEndpointGroup(&endpointGroup) + if err != nil { + log.Errorf("Error creating endpoint group: %+v, Err: %v", endpointGroup, err) + return err + } + + // Add the endpoint group to the list + service.EndpointGroups = append(service.EndpointGroups, dfltEpgName) + } + } + + // Link the service and endpoint group + for _, epgName := range service.EndpointGroups { + endpointGroup := contivModel.FindEndpointGroup(service.TenantName + ":" + epgName) + if endpointGroup == nil { + log.Errorf("Error: could not find endpoint group: %s", epgName) + return errors.New("could not find endpointGroup") + } + + // setup links + modeldb.AddLinkSet(&service.LinkSets.EndpointGroups, endpointGroup) + modeldb.AddLinkSet(&endpointGroup.LinkSets.Services, service) + + // save the endpointGroup + err = endpointGroup.Write() + if err != nil { + return err + } + } + + // Check if user specified any volume profile + if service.VolumeProfile == "" { + service.VolumeProfile = "default" + } + + volProfKey := service.TenantName + ":" + service.VolumeProfile + volProfile := contivModel.FindVolumeProfile(volProfKey) + if volProfile == nil { + log.Errorf("Could not find the volume profile: %s", service.VolumeProfile) + return errors.New("VolumeProfile not found") + } + + // fixup default values + if service.Scale == 0 { + service.Scale = 1 + } + + // Create service instances + for idx := int64(0); idx < service.Scale; idx++ { + instID := fmt.Sprintf("%d", idx+1) + var volumes []string + + // Create a volume for each instance based on the profile + if volProfile.DatastoreType != "none" { + instVolName := service.AppName + "." + service.ServiceName + "." + instID + err = contivModel.CreateVolume(&contivModel.Volume{ + Key: service.TenantName + ":" + instVolName, + VolumeName: instVolName, + TenantName: service.TenantName, + DatastoreType: volProfile.DatastoreType, + PoolName: volProfile.PoolName, + Size: volProfile.Size, + MountPoint: volProfile.MountPoint, + }) + if err != nil { + log.Errorf("Error creating volume %s. Err: %v", instVolName, err) + return err + } + volumes = []string{instVolName} + } + + // build instance params + instKey := service.TenantName + ":" + service.AppName + ":" + service.ServiceName + ":" + instID + inst := contivModel.ServiceInstance{ + Key: instKey, + InstanceID: instID, + TenantName: service.TenantName, + AppName: service.AppName, + ServiceName: service.ServiceName, + Volumes: volumes, + } + + // create the instance + err := contivModel.CreateServiceInstance(&inst) + if err != nil { + log.Errorf("Error creating service instance: %+v. Err: %v", inst, err) + return err + } + } + + return nil +} + +// ServiceUpdate updates service +func (ac *APIController) ServiceUpdate(service, params *contivModel.Service) error { + log.Infof("Received ServiceUpdate: %+v, params: %+v", service, params) + return nil +} + +// ServiceDelete deletes service +func (ac *APIController) ServiceDelete(service *contivModel.Service) error { + log.Infof("Received ServiceDelete: %+v", service) + return nil +} + +// ServiceInstanceCreate creates a service instance +func (ac *APIController) ServiceInstanceCreate(serviceInstance *contivModel.ServiceInstance) error { + log.Infof("Received ServiceInstanceCreate: %+v", serviceInstance) + inst := serviceInstance + + // Find the service + serviceKey := inst.TenantName + ":" + inst.AppName + ":" + inst.ServiceName + service := contivModel.FindService(serviceKey) + if service == nil { + log.Errorf("Service %s not found for instance: %+v", serviceKey, inst) + return errors.New("Service not found") + } + + // Add links + modeldb.AddLinkSet(&service.LinkSets.Instances, inst) + modeldb.AddLink(&inst.Links.Service, service) + + // setup links with volumes + for _, volumeName := range inst.Volumes { + // find the volume + volume := contivModel.FindVolume(inst.TenantName + ":" + volumeName) + if volume == nil { + log.Errorf("Could not find colume %s for service: %s", volumeName, inst.Key) + return errors.New("Could not find the volume") + } + + // add Links + modeldb.AddLinkSet(&inst.LinkSets.Volumes, volume) + modeldb.AddLinkSet(&volume.LinkSets.ServiceInstances, inst) + } + + return nil +} + +// ServiceInstanceUpdate updates a service instance +func (ac *APIController) ServiceInstanceUpdate(serviceInstance, params *contivModel.ServiceInstance) error { + log.Infof("Received ServiceInstanceUpdate: %+v, params: %+v", serviceInstance, params) + return nil +} + +// ServiceInstanceDelete deletes a service instance +func (ac *APIController) ServiceInstanceDelete(serviceInstance *contivModel.ServiceInstance) error { + log.Infof("Received ServiceInstanceDelete: %+v", serviceInstance) + return nil +} + +// TenantCreate creates a tenant +func (ac *APIController) TenantCreate(tenant *contivModel.Tenant) error { + log.Infof("Received TenantCreate: %+v", tenant) + + if tenant.TenantName == "" { + return errors.New("Invalid tenant name") + } + + // Get the state driver + stateDriver, err := utils.GetStateDriver() + if err != nil { + return err + } + + // Build tenant config + tenantCfg := intent.ConfigTenant{ + Name: tenant.TenantName, + DefaultNetType: "vlan", + SubnetPool: tenant.SubnetPool, + AllocSubnetLen: uint(tenant.SubnetLen), + VLANs: tenant.Vlans, + VXLANs: tenant.Vxlans, + } + + // Create the tenant + err = master.CreateTenant(stateDriver, &tenantCfg) + if err != nil { + log.Errorf("Error creating tenant {%+v}. Err: %v", tenant, err) + return err + } + + // Create private network for the tenant + err = contivModel.CreateNetwork(&contivModel.Network{ + Key: tenant.TenantName + ":" + "private", + IsPublic: false, + IsPrivate: true, + Encap: "vxlan", + Subnet: "10.1.0.0/16", + DefaultGw: "10.1.0.1", + NetworkName: "private", + TenantName: tenant.TenantName, + }) + if err != nil { + log.Errorf("Error creating privateNet for tenant: %+v. Err: %v", tenant, err) + return err + } + + // Create public network for the tenant + err = contivModel.CreateNetwork(&contivModel.Network{ + Key: tenant.TenantName + ":" + "public", + IsPublic: true, + IsPrivate: false, + Encap: "vlan", + Subnet: "192.168.1.0/24", + DefaultGw: "192.168.1.1", + NetworkName: "public", + TenantName: tenant.TenantName, + }) + if err != nil { + log.Errorf("Error creating publicNet for tenant: %+v. Err: %v", tenant, err) + return err + } + + // Create a default volume profile for the tenant + err = contivModel.CreateVolumeProfile(&contivModel.VolumeProfile{ + Key: tenant.TenantName + ":" + "default", + VolumeProfileName: "default", + TenantName: tenant.TenantName, + DatastoreType: "none", + PoolName: "", + Size: "", + MountPoint: "", + }) + if err != nil { + log.Errorf("Error creating default volume profile. Err: %v", err) + return err + } + + return nil +} + +// TenantUpdate updates a tenant +func (ac *APIController) TenantUpdate(tenant, params *contivModel.Tenant) error { + log.Infof("Received TenantUpdate: %+v, params: %+v", tenant, params) + + return errors.New("Cant change tenant parameters after its created") +} + +// TenantDelete deletes a tenant +func (ac *APIController) TenantDelete(tenant *contivModel.Tenant) error { + log.Infof("Received TenantDelete: %+v", tenant) + + // Get the state driver + stateDriver, err := utils.GetStateDriver() + if err != nil { + return err + } + + // FIXME: Should we walk all objects under the tenant and delete it? + + // Delete the tenant + err = master.DeleteTenantID(stateDriver, tenant.TenantName) + if err != nil { + log.Errorf("Error deleting tenant %s. Err: %v", tenant.TenantName, err) + } + + return nil +} + +// VolumeCreate creates a volume +func (ac *APIController) VolumeCreate(volume *contivModel.Volume) error { + log.Infof("Received VolumeCreate: %+v", volume) + + // Make sure tenant exists + if volume.TenantName == "" { + return errors.New("Invalid tenant name") + } + + tenant := contivModel.FindTenant(volume.TenantName) + if tenant == nil { + return errors.New("Tenant not found") + } + + // Setup links + modeldb.AddLink(&volume.Links.Tenant, tenant) + modeldb.AddLinkSet(&tenant.LinkSets.Volumes, volume) + + // Save the tenant too since we added the links + err := tenant.Write() + if err != nil { + return err + } + + return nil +} + +// VolumeUpdate updates a volume +func (ac *APIController) VolumeUpdate(volume, params *contivModel.Volume) error { + log.Infof("Received VolumeUpdate: %+v, params: %+v", volume, params) + return nil +} + +// VolumeDelete deletes a volume +func (ac *APIController) VolumeDelete(volume *contivModel.Volume) error { + log.Infof("Received VolumeDelete: %+v", volume) + return nil +} + +// VolumeProfileCreate create a volume profile +func (ac *APIController) VolumeProfileCreate(volumeProfile *contivModel.VolumeProfile) error { + log.Infof("Received VolumeProfileCreate: %+v", volumeProfile) + + // Make sure tenant exists + if volumeProfile.TenantName == "" { + return errors.New("Invalid tenant name") + } + tenant := contivModel.FindTenant(volumeProfile.TenantName) + if tenant == nil { + return errors.New("Tenant not found") + } + + // Setup links + modeldb.AddLink(&volumeProfile.Links.Tenant, tenant) + modeldb.AddLinkSet(&tenant.LinkSets.VolumeProfiles, volumeProfile) + + // Save the tenant too since we added the links + err := tenant.Write() + if err != nil { + return err + } + + return nil +} + +// VolumeProfileUpdate updates a volume profile +func (ac *APIController) VolumeProfileUpdate(volumeProfile, params *contivModel.VolumeProfile) error { + log.Infof("Received VolumeProfileUpdate: %+v, params: %+v", volumeProfile, params) + return nil +} + +// VolumeProfileDelete delete a volume profile +func (ac *APIController) VolumeProfileDelete(volumeProfile *contivModel.VolumeProfile) error { + return nil +} diff --git a/netplugin/netd.go b/netplugin/netd.go index f123b0104..8989a324e 100644 --- a/netplugin/netd.go +++ b/netplugin/netd.go @@ -34,6 +34,7 @@ import ( "github.com/contiv/netplugin/crtclient" "github.com/contiv/netplugin/crtclient/docker" "github.com/contiv/netplugin/drivers" + "github.com/contiv/netplugin/mgmtfn/dockplugin" "github.com/contiv/netplugin/netutils" "github.com/contiv/netplugin/plugin" "github.com/contiv/netplugin/utils" @@ -870,6 +871,10 @@ func main() { pluginConfig.Instance.VlanIntf = opts.vlanIntf } + // Initialize docker plugin + dockplugin.InitDockPlugin() + + // Init the driver plugins.. err = netPlugin.Init(pluginConfig, string(config)) if err != nil { log.Fatalf("Failed to initialize the plugin. Error: %s", err) diff --git a/scripts/dockerhost/start-dockerhosts b/scripts/dockerhost/start-dockerhosts index de9837df9..cbd68f09a 100755 --- a/scripts/dockerhost/start-dockerhosts +++ b/scripts/dockerhost/start-dockerhosts @@ -62,17 +62,23 @@ sudo ip link set br-em2 up num_nodes=1 if [ -n "$CONTIV_NODES" ]; -then +then num_nodes=$CONTIV_NODES fi echo "Num nodes = "$num_nodes # Pull the docker image for hosts from docker-hub # sudo docker build -t contiv/ubuntu $scriptdir -sudo docker pull contiv/ubuntu +sudo docker images contiv/ubuntu | grep ubuntu +if [ $? != 0 ]; then + sudo docker pull contiv/ubuntu +fi # Pull ubuntu image for use within the docker hosts -sudo docker pull ubuntu +sudo docker images ubuntu | grep ubuntu +if [ $? != 0 ]; then + sudo docker pull ubuntu +fi mkdir /tmp/ubuntu_image @@ -80,7 +86,7 @@ sudo docker save -o /tmp/ubuntu_image/ubuntu-image.tar ubuntu cluster="" first_node_ip="" -for i in `seq 1 $num_nodes`; +for i in `seq 1 $num_nodes`; do host="netplugin-node$i" echo "Starting $host" @@ -130,7 +136,7 @@ done sudo rm -rf /tmp/ubuntu_image -for i in `seq 1 $num_nodes`; +for i in `seq 1 $num_nodes`; do host="netplugin-node$i" echo "Starting etcd on $host" From cf55fa83019ac75ef460c393c544fefb92eaaeb3 Mon Sep 17 00:00:00 2001 From: Sukhesh Halemane Date: Thu, 20 Aug 2015 20:08:10 -0700 Subject: [PATCH 2/4] godeps import libnetwork Godeps changes --- Godeps/Godeps.json | 91 +- .../src/github.com/BurntSushi/toml/.gitignore | 5 + .../github.com/BurntSushi/toml/.travis.yml | 12 + .../src/github.com/BurntSushi/toml/COMPATIBLE | 3 + .../src/github.com/BurntSushi/toml/COPYING | 14 + .../src/github.com/BurntSushi/toml/Makefile | 19 + .../src/github.com/BurntSushi/toml/README.md | 220 ++ .../toml/cmd/toml-test-decoder/COPYING | 14 + .../toml/cmd/toml-test-decoder/README.md | 14 + .../toml/cmd/toml-test-decoder/main.go | 90 + .../toml/cmd/toml-test-encoder/COPYING | 14 + .../toml/cmd/toml-test-encoder/README.md | 14 + .../toml/cmd/toml-test-encoder/main.go | 131 + .../BurntSushi/toml/cmd/tomlv/COPYING | 14 + .../BurntSushi/toml/cmd/tomlv/README.md | 22 + .../BurntSushi/toml/cmd/tomlv/main.go | 61 + .../src/github.com/BurntSushi/toml/decode.go | 492 ++++ .../github.com/BurntSushi/toml/decode_meta.go | 122 + .../github.com/BurntSushi/toml/decode_test.go | 950 +++++++ .../src/github.com/BurntSushi/toml/doc.go | 27 + .../src/github.com/BurntSushi/toml/encode.go | 496 ++++ .../github.com/BurntSushi/toml/encode_test.go | 506 ++++ .../BurntSushi/toml/encoding_types.go | 19 + .../BurntSushi/toml/encoding_types_1.1.go | 18 + .../src/github.com/BurntSushi/toml/lex.go | 874 +++++++ .../src/github.com/BurntSushi/toml/parse.go | 498 ++++ .../github.com/BurntSushi/toml/session.vim | 1 + .../github.com/BurntSushi/toml/type_check.go | 91 + .../github.com/BurntSushi/toml/type_fields.go | 241 ++ .../github.com/armon/go-metrics/.gitignore | 22 + .../src/github.com/armon/go-metrics/LICENSE | 20 + .../src/github.com/armon/go-metrics/README.md | 68 + .../github.com/armon/go-metrics/const_unix.go | 12 + .../armon/go-metrics/const_windows.go | 13 + .../src/github.com/armon/go-metrics/inmem.go | 239 ++ .../armon/go-metrics/inmem_signal.go | 100 + .../armon/go-metrics/inmem_signal_test.go | 46 + .../github.com/armon/go-metrics/inmem_test.go | 95 + .../github.com/armon/go-metrics/metrics.go | 115 + .../armon/go-metrics/metrics_test.go | 262 ++ .../src/github.com/armon/go-metrics/sink.go | 52 + .../github.com/armon/go-metrics/sink_test.go | 120 + .../src/github.com/armon/go-metrics/start.go | 95 + .../github.com/armon/go-metrics/start_test.go | 110 + .../src/github.com/armon/go-metrics/statsd.go | 154 ++ .../armon/go-metrics/statsd_test.go | 105 + .../github.com/armon/go-metrics/statsite.go | 142 ++ .../armon/go-metrics/statsite_test.go | 101 + .../objdbClient.go => client/client.go} | 2 +- .../client_test.go} | 2 +- .../contiv/objmodel/objdb/modeldb/modeldb.go | 10 +- .../objmodel/objdb/{objdbApi.go => objdb.go} | 0 .../coreos/go-etcd/etcd/add_child.go | 23 + .../coreos/go-etcd/etcd/add_child_test.go | 73 + .../github.com/coreos/go-etcd/etcd/client.go | 481 ++++ .../coreos/go-etcd/etcd/client_test.go | 108 + .../github.com/coreos/go-etcd/etcd/cluster.go | 37 + .../coreos/go-etcd/etcd/compare_and_delete.go | 34 + .../go-etcd/etcd/compare_and_delete_test.go | 46 + .../coreos/go-etcd/etcd/compare_and_swap.go | 36 + .../go-etcd/etcd/compare_and_swap_test.go | 57 + .../github.com/coreos/go-etcd/etcd/debug.go | 55 + .../coreos/go-etcd/etcd/debug_test.go | 28 + .../github.com/coreos/go-etcd/etcd/delete.go | 40 + .../coreos/go-etcd/etcd/delete_test.go | 81 + .../github.com/coreos/go-etcd/etcd/error.go | 49 + .../src/github.com/coreos/go-etcd/etcd/get.go | 32 + .../coreos/go-etcd/etcd/get_test.go | 131 + .../github.com/coreos/go-etcd/etcd/member.go | 30 + .../coreos/go-etcd/etcd/member_test.go | 71 + .../github.com/coreos/go-etcd/etcd/options.go | 72 + .../coreos/go-etcd/etcd/requests.go | 403 +++ .../coreos/go-etcd/etcd/requests_test.go | 22 + .../coreos/go-etcd/etcd/response.go | 89 + .../coreos/go-etcd/etcd/set_curl_chan_test.go | 42 + .../coreos/go-etcd/etcd/set_update_create.go | 137 ++ .../go-etcd/etcd/set_update_create_test.go | 241 ++ .../github.com/coreos/go-etcd/etcd/version.go | 6 + .../github.com/coreos/go-etcd/etcd/watch.go | 103 + .../coreos/go-etcd/etcd/watch_test.go | 119 + .../docker/docker/pkg/ioutils/fmt.go | 14 + .../docker/docker/pkg/ioutils/fmt_test.go | 17 + .../docker/docker/pkg/ioutils/multireader.go | 226 ++ .../docker/pkg/ioutils/multireader_test.go | 149 ++ .../docker/docker/pkg/ioutils/readers.go | 262 ++ .../docker/docker/pkg/ioutils/readers_test.go | 216 ++ .../docker/docker/pkg/ioutils/scheduler.go | 6 + .../docker/pkg/ioutils/scheduler_gccgo.go | 13 + .../docker/docker/pkg/ioutils/writeflusher.go | 51 + .../docker/docker/pkg/ioutils/writers.go | 66 + .../docker/docker/pkg/ioutils/writers_test.go | 65 + .../docker/pkg/parsers/kernel/kernel.go | 100 + .../docker/pkg/parsers/kernel/kernel_test.go | 92 + .../pkg/parsers/kernel/kernel_windows.go | 67 + .../docker/pkg/parsers/kernel/uname_linux.go | 19 + .../pkg/parsers/kernel/uname_unsupported.go | 18 + .../docker/pkg/proxy/network_proxy_test.go | 216 ++ .../docker/docker/pkg/proxy/proxy.go | 37 + .../docker/docker/pkg/proxy/stub_proxy.go | 31 + .../docker/docker/pkg/proxy/tcp_proxy.go | 99 + .../docker/docker/pkg/proxy/udp_proxy.go | 169 ++ .../docker/docker/pkg/random/random.go | 71 + .../docker/docker/pkg/random/random_test.go | 22 + .../docker/docker/pkg/reexec/README.md | 5 + .../docker/pkg/reexec/command_freebsd.go | 23 + .../docker/docker/pkg/reexec/command_linux.go | 28 + .../docker/pkg/reexec/command_unsupported.go | 12 + .../docker/pkg/reexec/command_windows.go | 23 + .../docker/docker/pkg/reexec/reexec.go | 47 + .../docker/docker/pkg/stringid/README.md | 1 + .../docker/docker/pkg/stringid/stringid.go | 67 + .../docker/pkg/stringid/stringid_test.go | 56 + .../src/github.com/docker/libkv/.travis.yml | 34 + .../src/github.com/docker/libkv/LICENSE | 191 ++ .../src/github.com/docker/libkv/README.md | 108 + .../src/github.com/docker/libkv/libkv.go | 29 + .../src/github.com/docker/libkv/libkv_test.go | 80 + .../github.com/docker/libkv/script/.validate | 33 + .../github.com/docker/libkv/script/coverage | 21 + .../docker/libkv/script/travis_consul.sh | 18 + .../docker/libkv/script/travis_etcd.sh | 11 + .../docker/libkv/script/travis_zk.sh | 12 + .../docker/libkv/script/validate-gofmt | 30 + .../docker/libkv/store/consul/consul.go | 422 ++++ .../docker/libkv/store/consul/consul_test.go | 62 + .../docker/libkv/store/etcd/etcd.go | 499 ++++ .../docker/libkv/store/etcd/etcd_test.go | 34 + .../github.com/docker/libkv/store/helpers.go | 47 + .../docker/libkv/store/mock/mock.go | 113 + .../github.com/docker/libkv/store/store.go | 119 + .../docker/libkv/store/zookeeper/zookeeper.go | 379 +++ .../libkv/store/zookeeper/zookeeper_test.go | 34 + .../docker/libkv/testutils/utils.go | 430 ++++ .../github.com/docker/libnetwork/.gitignore | 33 + .../docker/libnetwork/Godeps/Godeps.json | 137 ++ .../docker/libnetwork/Godeps/Readme | 5 + .../src/github.com/docker/libnetwork/LICENSE | 202 ++ .../github.com/docker/libnetwork/MAINTAINERS | 5 + .../src/github.com/docker/libnetwork/Makefile | 79 + .../github.com/docker/libnetwork/README.md | 88 + .../github.com/docker/libnetwork/ROADMAP.md | 20 + .../github.com/docker/libnetwork/api/api.go | 796 ++++++ .../docker/libnetwork/api/api_test.go | 2183 +++++++++++++++++ .../github.com/docker/libnetwork/api/types.go | 81 + .../docker/libnetwork/bitseq/sequence.go | 541 ++++ .../docker/libnetwork/bitseq/sequence_test.go | 497 ++++ .../docker/libnetwork/bitseq/store.go | 138 ++ .../github.com/docker/libnetwork/circle.yml | 12 + .../docker/libnetwork/client/client.go | 115 + .../libnetwork/client/client_service_test.go | 122 + .../docker/libnetwork/client/client_test.go | 217 ++ .../docker/libnetwork/client/network.go | 231 ++ .../docker/libnetwork/client/service.go | 362 +++ .../docker/libnetwork/client/types.go | 73 + .../docker/libnetwork/cmd/dnet/dnet.go | 288 +++ .../docker/libnetwork/cmd/dnet/dnet_test.go | 132 + .../docker/libnetwork/cmd/dnet/flags.go | 51 + .../libnetwork/cmd/dnet/libnetwork.toml | 12 + .../libnetwork/cmd/ovrouter/ovrouter.go | 149 ++ .../libnetwork/cmd/readme_test/readme.go | 65 + .../libnetwork/cmd/test/libnetwork.toml | 12 + .../docker/libnetwork/cmd/test/main.go | 49 + .../docker/libnetwork/config/config.go | 116 + .../docker/libnetwork/config/config_test.go | 55 + .../docker/libnetwork/config/libnetwork.toml | 12 + .../docker/libnetwork/controller.go | 399 +++ .../docker/libnetwork/datastore/datastore.go | 194 ++ .../libnetwork/datastore/datastore_test.go | 241 ++ .../docker/libnetwork/datastore/mock_store.go | 129 + .../docker/libnetwork/docs/Vagrantfile | 58 + .../docker/libnetwork/docs/bridge.md | 13 + .../docker/libnetwork/docs/design.md | 158 ++ .../docker/libnetwork/docs/legacy.md | 15 + .../docker/libnetwork/docs/overlay.md | 153 ++ .../docker/libnetwork/docs/remote.md | 203 ++ .../docs/vagrant-systemd/docker.service | 16 + .../docker/libnetwork/docs/vagrant.md | 185 ++ .../docker/libnetwork/driverapi/driverapi.go | 137 ++ .../docker/libnetwork/driverapi/errors.go | 56 + .../libnetwork/drivers/bridge/bridge.go | 1417 +++++++++++ .../libnetwork/drivers/bridge/bridge_test.go | 688 ++++++ .../libnetwork/drivers/bridge/errors.go | 330 +++ .../libnetwork/drivers/bridge/interface.go | 63 + .../drivers/bridge/interface_test.go | 33 + .../docker/libnetwork/drivers/bridge/link.go | 85 + .../libnetwork/drivers/bridge/link_test.go | 39 + .../bridge/netlink_deprecated_linux.go | 131 + .../netlink_deprecated_linux_armppc64.go | 7 + .../bridge/netlink_deprecated_linux_notarm.go | 7 + .../bridge/netlink_deprecated_unsupported.go | 18 + .../libnetwork/drivers/bridge/network_test.go | 219 ++ .../libnetwork/drivers/bridge/port_mapping.go | 128 + .../drivers/bridge/port_mapping_test.go | 80 + .../libnetwork/drivers/bridge/resolvconf.go | 67 + .../drivers/bridge/resolvconf_test.go | 53 + .../docker/libnetwork/drivers/bridge/setup.go | 26 + .../bridge/setup_bridgenetfiltering.go | 162 ++ .../bridge/setup_bridgenetfiltering_test.go | 12 + .../libnetwork/drivers/bridge/setup_device.go | 66 + .../drivers/bridge/setup_device_test.go | 75 + .../drivers/bridge/setup_firewalld.go | 20 + .../drivers/bridge/setup_fixedcidrv4.go | 19 + .../drivers/bridge/setup_fixedcidrv4_test.go | 62 + .../drivers/bridge/setup_fixedcidrv6.go | 27 + .../drivers/bridge/setup_fixedcidrv6_test.go | 44 + .../drivers/bridge/setup_ip_forwarding.go | 29 + .../bridge/setup_ip_forwarding_test.go | 51 + .../drivers/bridge/setup_ip_tables.go | 253 ++ .../drivers/bridge/setup_ip_tables_test.go | 124 + .../libnetwork/drivers/bridge/setup_ipv4.go | 146 ++ .../drivers/bridge/setup_ipv4_test.go | 111 + .../libnetwork/drivers/bridge/setup_ipv6.go | 106 + .../drivers/bridge/setup_ipv6_test.go | 70 + .../libnetwork/drivers/bridge/setup_verify.go | 46 + .../drivers/bridge/setup_verify_test.go | 110 + .../docker/libnetwork/drivers/host/host.go | 74 + .../libnetwork/drivers/host/host_test.go | 50 + .../docker/libnetwork/drivers/null/null.go | 70 + .../libnetwork/drivers/null/null_test.go | 50 + .../libnetwork/drivers/overlay/joinleave.go | 99 + .../libnetwork/drivers/overlay/ov_endpoint.go | 110 + .../libnetwork/drivers/overlay/ov_network.go | 367 +++ .../libnetwork/drivers/overlay/ov_serf.go | 249 ++ .../libnetwork/drivers/overlay/ov_utils.go | 82 + .../libnetwork/drivers/overlay/overlay.go | 158 ++ .../drivers/overlay/overlay_test.go | 130 + .../libnetwork/drivers/overlay/peerdb.go | 284 +++ .../libnetwork/drivers/remote/driver.go | 293 +++ .../libnetwork/drivers/remote/driver_test.go | 436 ++++ .../libnetwork/drivers/windows/windows.go | 58 + .../docker/libnetwork/drivers_freebsd.go | 19 + .../docker/libnetwork/drivers_linux.go | 25 + .../docker/libnetwork/drivers_windows.go | 17 + .../github.com/docker/libnetwork/endpoint.go | 994 ++++++++ .../docker/libnetwork/endpoint_info.go | 323 +++ .../src/github.com/docker/libnetwork/error.go | 175 ++ .../docker/libnetwork/errors_test.go | 51 + .../docker/libnetwork/etchosts/etchosts.go | 127 + .../libnetwork/etchosts/etchosts_test.go | 249 ++ .../libnetwork/hostdiscovery/hostdiscovery.go | 154 ++ .../hostdiscovery/hostdiscovery_api.go | 23 + .../hostdiscovery/hostdiscovery_disabled.go | 28 + .../hostdiscovery/hostdiscovery_test.go | 142 ++ .../libnetwork/hostdiscovery/libnetwork.toml | 6 + .../github.com/docker/libnetwork/idm/idm.go | 60 + .../docker/libnetwork/idm/idm_test.go | 110 + .../libnetwork/ipallocator/allocator.go | 175 ++ .../libnetwork/ipallocator/allocator_test.go | 690 ++++++ .../docker/libnetwork/ipam/allocator.go | 624 +++++ .../docker/libnetwork/ipam/allocator_test.go | 612 +++++ .../docker/libnetwork/ipam/contract.go | 100 + .../docker/libnetwork/ipam/store.go | 178 ++ .../docker/libnetwork/iptables/firewalld.go | 164 ++ .../libnetwork/iptables/firewalld_test.go | 86 + .../docker/libnetwork/iptables/iptables.go | 337 +++ .../libnetwork/iptables/iptables_test.go | 236 ++ .../libnetwork/libnetwork_internal_test.go | 32 + .../docker/libnetwork/libnetwork_test.go | 2115 ++++++++++++++++ .../docker/libnetwork/netlabel/labels.go | 52 + .../docker/libnetwork/netutils/test_utils.go | 41 + .../docker/libnetwork/netutils/utils.go | 172 ++ .../docker/libnetwork/netutils/utils_test.go | 211 ++ .../github.com/docker/libnetwork/network.go | 471 ++++ .../docker/libnetwork/options/options.go | 73 + .../docker/libnetwork/options/options_test.go | 97 + .../libnetwork/portallocator/portallocator.go | 270 ++ .../portallocator/portallocator_test.go | 320 +++ .../docker/libnetwork/portmapper/mapper.go | 226 ++ .../libnetwork/portmapper/mapper_test.go | 270 ++ .../libnetwork/portmapper/mock_proxy.go | 18 + .../docker/libnetwork/portmapper/proxy.go | 209 ++ .../docker/libnetwork/resolvconf/README.md | 1 + .../libnetwork/resolvconf/dns/resolvconf.go | 17 + .../libnetwork/resolvconf/resolvconf.go | 187 ++ .../libnetwork/resolvconf/resolvconf_test.go | 240 ++ .../libnetwork/sandbox/interface_freebsd.go | 4 + .../libnetwork/sandbox/interface_linux.go | 370 +++ .../libnetwork/sandbox/interface_windows.go | 4 + .../libnetwork/sandbox/namespace_linux.go | 303 +++ .../sandbox/namespace_unsupported.go | 8 + .../libnetwork/sandbox/namespace_windows.go | 23 + .../libnetwork/sandbox/neigh_freebsd.go | 4 + .../docker/libnetwork/sandbox/neigh_linux.go | 138 ++ .../libnetwork/sandbox/neigh_windows.go | 4 + .../libnetwork/sandbox/options_linux.go | 61 + .../docker/libnetwork/sandbox/route_linux.go | 197 ++ .../docker/libnetwork/sandbox/sandbox.go | 170 ++ .../libnetwork/sandbox/sandbox_freebsd.go | 23 + .../libnetwork/sandbox/sandbox_linux_test.go | 180 ++ .../docker/libnetwork/sandbox/sandbox_test.go | 190 ++ .../libnetwork/sandbox/sandbox_unsupported.go | 22 + .../sandbox/sandbox_unsupported_test.go | 20 + .../docker/libnetwork/sandboxdata.go | 259 ++ .../docker/libnetwork/sandboxdata_test.go | 141 ++ .../src/github.com/docker/libnetwork/store.go | 371 +++ .../libnetwork/test/integration/README.md | 34 + .../test/integration/daemon-configs.bats | 104 + .../libnetwork/test/integration/daemon.cfg | 4 + .../libnetwork/test/integration/helpers.bash | 50 + .../docker/libnetwork/types/types.go | 463 ++++ .../docker/libnetwork/types/types_test.go | 170 ++ .../github.com/godbus/dbus/CONTRIBUTING.md | 50 + .../src/github.com/godbus/dbus/LICENSE | 25 + .../src/github.com/godbus/dbus/MAINTAINERS | 2 + .../github.com/godbus/dbus/README.markdown | 38 + .../src/github.com/godbus/dbus/auth.go | 253 ++ .../github.com/godbus/dbus/auth_external.go | 26 + .../src/github.com/godbus/dbus/auth_sha1.go | 102 + .../src/github.com/godbus/dbus/call.go | 147 ++ .../src/github.com/godbus/dbus/conn.go | 609 +++++ .../src/github.com/godbus/dbus/conn_darwin.go | 21 + .../src/github.com/godbus/dbus/conn_other.go | 27 + .../src/github.com/godbus/dbus/conn_test.go | 199 ++ .../src/github.com/godbus/dbus/dbus.go | 258 ++ .../src/github.com/godbus/dbus/decoder.go | 228 ++ .../src/github.com/godbus/dbus/doc.go | 63 + .../src/github.com/godbus/dbus/encoder.go | 208 ++ .../github.com/godbus/dbus/encoder_test.go | 58 + .../github.com/godbus/dbus/examples_test.go | 50 + .../src/github.com/godbus/dbus/export.go | 302 +++ .../src/github.com/godbus/dbus/homedir.go | 28 + .../github.com/godbus/dbus/homedir_dynamic.go | 15 + .../github.com/godbus/dbus/homedir_static.go | 45 + .../github.com/godbus/dbus/introspect/call.go | 27 + .../godbus/dbus/introspect/introspect.go | 86 + .../godbus/dbus/introspect/introspectable.go | 75 + .../src/github.com/godbus/dbus/message.go | 346 +++ .../src/github.com/godbus/dbus/prop/prop.go | 264 ++ .../src/github.com/godbus/dbus/proto_test.go | 369 +++ .../src/github.com/godbus/dbus/sig.go | 257 ++ .../src/github.com/godbus/dbus/sig_test.go | 70 + .../godbus/dbus/transport_darwin.go | 6 + .../godbus/dbus/transport_generic.go | 35 + .../github.com/godbus/dbus/transport_unix.go | 196 ++ .../godbus/dbus/transport_unix_test.go | 49 + .../dbus/transport_unixcred_dragonfly.go | 95 + .../godbus/dbus/transport_unixcred_linux.go | 25 + .../src/github.com/godbus/dbus/variant.go | 139 ++ .../github.com/godbus/dbus/variant_lexer.go | 284 +++ .../github.com/godbus/dbus/variant_parser.go | 817 ++++++ .../github.com/godbus/dbus/variant_test.go | 78 + .../hashicorp/go-msgpack/codec/0doc.go | 143 ++ .../hashicorp/go-msgpack/codec/README.md | 174 ++ .../hashicorp/go-msgpack/codec/bench_test.go | 319 +++ .../hashicorp/go-msgpack/codec/binc.go | 786 ++++++ .../hashicorp/go-msgpack/codec/codecs_test.go | 1002 ++++++++ .../hashicorp/go-msgpack/codec/decode.go | 1048 ++++++++ .../hashicorp/go-msgpack/codec/encode.go | 1001 ++++++++ .../go-msgpack/codec/ext_dep_test.go | 75 + .../hashicorp/go-msgpack/codec/helper.go | 589 +++++ .../go-msgpack/codec/helper_internal.go | 127 + .../hashicorp/go-msgpack/codec/msgpack.go | 816 ++++++ .../go-msgpack/codec/msgpack_test.py | 110 + .../hashicorp/go-msgpack/codec/rpc.go | 152 ++ .../hashicorp/go-msgpack/codec/simple.go | 461 ++++ .../hashicorp/go-msgpack/codec/time.go | 193 ++ .../go-msgpack/codec/z_helper_test.go | 103 + .../hashicorp/memberlist/.gitignore | 25 + .../github.com/hashicorp/memberlist/LICENSE | 354 +++ .../github.com/hashicorp/memberlist/Makefile | 14 + .../github.com/hashicorp/memberlist/README.md | 137 ++ .../hashicorp/memberlist/broadcast.go | 100 + .../hashicorp/memberlist/broadcast_test.go | 27 + .../github.com/hashicorp/memberlist/config.go | 209 ++ .../hashicorp/memberlist/conflict_delegate.go | 10 + .../hashicorp/memberlist/delegate.go | 36 + .../hashicorp/memberlist/event_delegate.go | 61 + .../hashicorp/memberlist/integ_test.go | 89 + .../hashicorp/memberlist/keyring.go | 144 ++ .../hashicorp/memberlist/keyring_test.go | 154 ++ .../hashicorp/memberlist/memberlist.go | 525 ++++ .../hashicorp/memberlist/memberlist_test.go | 1144 +++++++++ .../hashicorp/memberlist/merge_delegate.go | 13 + .../github.com/hashicorp/memberlist/net.go | 852 +++++++ .../hashicorp/memberlist/net_test.go | 476 ++++ .../github.com/hashicorp/memberlist/queue.go | 167 ++ .../hashicorp/memberlist/queue_test.go | 172 ++ .../hashicorp/memberlist/security.go | 198 ++ .../hashicorp/memberlist/security_test.go | 70 + .../github.com/hashicorp/memberlist/state.go | 916 +++++++ .../hashicorp/memberlist/state_test.go | 1039 ++++++++ .../hashicorp/memberlist/test/setup_subnet.sh | 28 + .../github.com/hashicorp/memberlist/todo.md | 6 + .../github.com/hashicorp/memberlist/util.go | 325 +++ .../hashicorp/memberlist/util_test.go | 293 +++ .../hashicorp/serf/serf/broadcast.go | 27 + .../hashicorp/serf/serf/broadcast_test.go | 32 + .../hashicorp/serf/serf/coalesce.go | 80 + .../hashicorp/serf/serf/coalesce_member.go | 68 + .../serf/serf/coalesce_member_test.go | 185 ++ .../hashicorp/serf/serf/coalesce_test.go | 140 ++ .../hashicorp/serf/serf/coalesce_user.go | 52 + .../hashicorp/serf/serf/coalesce_user_test.go | 102 + .../github.com/hashicorp/serf/serf/config.go | 234 ++ .../hashicorp/serf/serf/config_test.go | 12 + .../hashicorp/serf/serf/conflict_delegate.go | 13 + .../hashicorp/serf/serf/delegate.go | 247 ++ .../hashicorp/serf/serf/delegate_test.go | 213 ++ .../github.com/hashicorp/serf/serf/event.go | 168 ++ .../hashicorp/serf/serf/event_delegate.go | 21 + .../hashicorp/serf/serf/event_test.go | 211 ++ .../hashicorp/serf/serf/internal_query.go | 312 +++ .../serf/serf/internal_query_test.go | 89 + .../hashicorp/serf/serf/keymanager.go | 166 ++ .../hashicorp/serf/serf/keymanager_test.go | 272 ++ .../github.com/hashicorp/serf/serf/lamport.go | 45 + .../hashicorp/serf/serf/lamport_test.go | 39 + .../hashicorp/serf/serf/merge_delegate.go | 35 + .../hashicorp/serf/serf/messages.go | 147 ++ .../hashicorp/serf/serf/messages_test.go | 58 + .../github.com/hashicorp/serf/serf/query.go | 210 ++ .../hashicorp/serf/serf/query_test.go | 138 ++ .../github.com/hashicorp/serf/serf/serf.go | 1598 ++++++++++++ .../serf/serf/serf_internals_test.go | 442 ++++ .../hashicorp/serf/serf/serf_test.go | 1647 +++++++++++++ .../hashicorp/serf/serf/snapshot.go | 500 ++++ .../hashicorp/serf/serf/snapshot_test.go | 374 +++ .../samuel/go-zookeeper/zk/cluster_test.go | 166 ++ .../github.com/samuel/go-zookeeper/zk/conn.go | 844 +++++++ .../samuel/go-zookeeper/zk/constants.go | 242 ++ .../samuel/go-zookeeper/zk/constants_test.go | 24 + .../github.com/samuel/go-zookeeper/zk/flw.go | 288 +++ .../samuel/go-zookeeper/zk/flw_test.go | 367 +++ .../github.com/samuel/go-zookeeper/zk/lock.go | 131 + .../samuel/go-zookeeper/zk/lock_test.go | 94 + .../samuel/go-zookeeper/zk/server_help.go | 119 + .../samuel/go-zookeeper/zk/server_java.go | 136 + .../samuel/go-zookeeper/zk/structs.go | 633 +++++ .../samuel/go-zookeeper/zk/structs_test.go | 60 + .../samuel/go-zookeeper/zk/tracer.go | 148 ++ .../github.com/samuel/go-zookeeper/zk/util.go | 54 + .../samuel/go-zookeeper/zk/util_test.go | 17 + .../samuel/go-zookeeper/zk/zk_test.go | 518 ++++ .../github.com/vishvananda/netlink/README.md | 8 +- .../github.com/vishvananda/netlink/addr.go | 4 +- .../vishvananda/netlink/addr_linux.go | 18 +- .../github.com/vishvananda/netlink/link.go | 10 +- .../vishvananda/netlink/link_linux.go | 48 +- .../vishvananda/netlink/link_test.go | 107 +- .../vishvananda/netlink/neigh_linux.go | 2 +- .../vishvananda/netlink/nl/link_linux.go | 10 +- .../vishvananda/netlink/nl/nl_linux.go | 25 +- .../vishvananda/netlink/nl/route_linux.go | 9 + .../vishvananda/netlink/nl/xfrm_linux.go | 7 +- .../vishvananda/netlink/protinfo.go | 2 +- .../vishvananda/netlink/route_linux.go | 11 +- .../vishvananda/netlink/xfrm_policy_linux.go | 2 +- .../vishvananda/netlink/xfrm_state_linux.go | 2 +- .../src/github.com/vishvananda/netns/LICENSE | 192 ++ .../github.com/vishvananda/netns/README.md | 49 + .../src/github.com/vishvananda/netns/netns.go | 67 + .../vishvananda/netns/netns_linux.go | 205 ++ .../vishvananda/netns/netns_linux_386.go | 7 + .../vishvananda/netns/netns_linux_amd64.go | 7 + .../vishvananda/netns/netns_linux_arm.go | 7 + .../vishvananda/netns/netns_linux_arm64.go | 7 + .../vishvananda/netns/netns_linux_ppc64le.go | 7 + .../vishvananda/netns/netns_test.go | 66 + .../vishvananda/netns/netns_unspecified.go | 35 + 459 files changed, 78119 insertions(+), 79 deletions(-) create mode 100644 Godeps/_workspace/src/github.com/BurntSushi/toml/.gitignore create mode 100644 Godeps/_workspace/src/github.com/BurntSushi/toml/.travis.yml create mode 100644 Godeps/_workspace/src/github.com/BurntSushi/toml/COMPATIBLE create mode 100644 Godeps/_workspace/src/github.com/BurntSushi/toml/COPYING create mode 100644 Godeps/_workspace/src/github.com/BurntSushi/toml/Makefile create mode 100644 Godeps/_workspace/src/github.com/BurntSushi/toml/README.md create mode 100644 Godeps/_workspace/src/github.com/BurntSushi/toml/cmd/toml-test-decoder/COPYING create mode 100644 Godeps/_workspace/src/github.com/BurntSushi/toml/cmd/toml-test-decoder/README.md create mode 100644 Godeps/_workspace/src/github.com/BurntSushi/toml/cmd/toml-test-decoder/main.go create mode 100644 Godeps/_workspace/src/github.com/BurntSushi/toml/cmd/toml-test-encoder/COPYING create mode 100644 Godeps/_workspace/src/github.com/BurntSushi/toml/cmd/toml-test-encoder/README.md create mode 100644 Godeps/_workspace/src/github.com/BurntSushi/toml/cmd/toml-test-encoder/main.go create mode 100644 Godeps/_workspace/src/github.com/BurntSushi/toml/cmd/tomlv/COPYING create mode 100644 Godeps/_workspace/src/github.com/BurntSushi/toml/cmd/tomlv/README.md create mode 100644 Godeps/_workspace/src/github.com/BurntSushi/toml/cmd/tomlv/main.go create mode 100644 Godeps/_workspace/src/github.com/BurntSushi/toml/decode.go create mode 100644 Godeps/_workspace/src/github.com/BurntSushi/toml/decode_meta.go create mode 100644 Godeps/_workspace/src/github.com/BurntSushi/toml/decode_test.go create mode 100644 Godeps/_workspace/src/github.com/BurntSushi/toml/doc.go create mode 100644 Godeps/_workspace/src/github.com/BurntSushi/toml/encode.go create mode 100644 Godeps/_workspace/src/github.com/BurntSushi/toml/encode_test.go create mode 100644 Godeps/_workspace/src/github.com/BurntSushi/toml/encoding_types.go create mode 100644 Godeps/_workspace/src/github.com/BurntSushi/toml/encoding_types_1.1.go create mode 100644 Godeps/_workspace/src/github.com/BurntSushi/toml/lex.go create mode 100644 Godeps/_workspace/src/github.com/BurntSushi/toml/parse.go create mode 100644 Godeps/_workspace/src/github.com/BurntSushi/toml/session.vim create mode 100644 Godeps/_workspace/src/github.com/BurntSushi/toml/type_check.go create mode 100644 Godeps/_workspace/src/github.com/BurntSushi/toml/type_fields.go create mode 100644 Godeps/_workspace/src/github.com/armon/go-metrics/.gitignore create mode 100644 Godeps/_workspace/src/github.com/armon/go-metrics/LICENSE create mode 100644 Godeps/_workspace/src/github.com/armon/go-metrics/README.md create mode 100644 Godeps/_workspace/src/github.com/armon/go-metrics/const_unix.go create mode 100644 Godeps/_workspace/src/github.com/armon/go-metrics/const_windows.go create mode 100644 Godeps/_workspace/src/github.com/armon/go-metrics/inmem.go create mode 100644 Godeps/_workspace/src/github.com/armon/go-metrics/inmem_signal.go create mode 100644 Godeps/_workspace/src/github.com/armon/go-metrics/inmem_signal_test.go create mode 100644 Godeps/_workspace/src/github.com/armon/go-metrics/inmem_test.go create mode 100644 Godeps/_workspace/src/github.com/armon/go-metrics/metrics.go create mode 100644 Godeps/_workspace/src/github.com/armon/go-metrics/metrics_test.go create mode 100644 Godeps/_workspace/src/github.com/armon/go-metrics/sink.go create mode 100644 Godeps/_workspace/src/github.com/armon/go-metrics/sink_test.go create mode 100644 Godeps/_workspace/src/github.com/armon/go-metrics/start.go create mode 100644 Godeps/_workspace/src/github.com/armon/go-metrics/start_test.go create mode 100644 Godeps/_workspace/src/github.com/armon/go-metrics/statsd.go create mode 100644 Godeps/_workspace/src/github.com/armon/go-metrics/statsd_test.go create mode 100644 Godeps/_workspace/src/github.com/armon/go-metrics/statsite.go create mode 100644 Godeps/_workspace/src/github.com/armon/go-metrics/statsite_test.go rename Godeps/_workspace/src/github.com/contiv/objmodel/objdb/{objdbClient/objdbClient.go => client/client.go} (96%) rename Godeps/_workspace/src/github.com/contiv/objmodel/objdb/{objdbClient/objdbClient_test.go => client/client_test.go} (99%) rename Godeps/_workspace/src/github.com/contiv/objmodel/objdb/{objdbApi.go => objdb.go} (100%) create mode 100644 Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/add_child.go create mode 100644 Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/add_child_test.go create mode 100644 Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/client.go create mode 100644 Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/client_test.go create mode 100644 Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/cluster.go create mode 100644 Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/compare_and_delete.go create mode 100644 Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/compare_and_delete_test.go create mode 100644 Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/compare_and_swap.go create mode 100644 Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/compare_and_swap_test.go create mode 100644 Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/debug.go create mode 100644 Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/debug_test.go create mode 100644 Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/delete.go create mode 100644 Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/delete_test.go create mode 100644 Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/error.go create mode 100644 Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/get.go create mode 100644 Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/get_test.go create mode 100644 Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/member.go create mode 100644 Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/member_test.go create mode 100644 Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/options.go create mode 100644 Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/requests.go create mode 100644 Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/requests_test.go create mode 100644 Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/response.go create mode 100644 Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/set_curl_chan_test.go create mode 100644 Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/set_update_create.go create mode 100644 Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/set_update_create_test.go create mode 100644 Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/version.go create mode 100644 Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/watch.go create mode 100644 Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/watch_test.go create mode 100644 Godeps/_workspace/src/github.com/docker/docker/pkg/ioutils/fmt.go create mode 100644 Godeps/_workspace/src/github.com/docker/docker/pkg/ioutils/fmt_test.go create mode 100644 Godeps/_workspace/src/github.com/docker/docker/pkg/ioutils/multireader.go create mode 100644 Godeps/_workspace/src/github.com/docker/docker/pkg/ioutils/multireader_test.go create mode 100644 Godeps/_workspace/src/github.com/docker/docker/pkg/ioutils/readers.go create mode 100644 Godeps/_workspace/src/github.com/docker/docker/pkg/ioutils/readers_test.go create mode 100644 Godeps/_workspace/src/github.com/docker/docker/pkg/ioutils/scheduler.go create mode 100644 Godeps/_workspace/src/github.com/docker/docker/pkg/ioutils/scheduler_gccgo.go create mode 100644 Godeps/_workspace/src/github.com/docker/docker/pkg/ioutils/writeflusher.go create mode 100644 Godeps/_workspace/src/github.com/docker/docker/pkg/ioutils/writers.go create mode 100644 Godeps/_workspace/src/github.com/docker/docker/pkg/ioutils/writers_test.go create mode 100644 Godeps/_workspace/src/github.com/docker/docker/pkg/parsers/kernel/kernel.go create mode 100644 Godeps/_workspace/src/github.com/docker/docker/pkg/parsers/kernel/kernel_test.go create mode 100644 Godeps/_workspace/src/github.com/docker/docker/pkg/parsers/kernel/kernel_windows.go create mode 100644 Godeps/_workspace/src/github.com/docker/docker/pkg/parsers/kernel/uname_linux.go create mode 100644 Godeps/_workspace/src/github.com/docker/docker/pkg/parsers/kernel/uname_unsupported.go create mode 100644 Godeps/_workspace/src/github.com/docker/docker/pkg/proxy/network_proxy_test.go create mode 100644 Godeps/_workspace/src/github.com/docker/docker/pkg/proxy/proxy.go create mode 100644 Godeps/_workspace/src/github.com/docker/docker/pkg/proxy/stub_proxy.go create mode 100644 Godeps/_workspace/src/github.com/docker/docker/pkg/proxy/tcp_proxy.go create mode 100644 Godeps/_workspace/src/github.com/docker/docker/pkg/proxy/udp_proxy.go create mode 100644 Godeps/_workspace/src/github.com/docker/docker/pkg/random/random.go create mode 100644 Godeps/_workspace/src/github.com/docker/docker/pkg/random/random_test.go create mode 100644 Godeps/_workspace/src/github.com/docker/docker/pkg/reexec/README.md create mode 100644 Godeps/_workspace/src/github.com/docker/docker/pkg/reexec/command_freebsd.go create mode 100644 Godeps/_workspace/src/github.com/docker/docker/pkg/reexec/command_linux.go create mode 100644 Godeps/_workspace/src/github.com/docker/docker/pkg/reexec/command_unsupported.go create mode 100644 Godeps/_workspace/src/github.com/docker/docker/pkg/reexec/command_windows.go create mode 100644 Godeps/_workspace/src/github.com/docker/docker/pkg/reexec/reexec.go create mode 100644 Godeps/_workspace/src/github.com/docker/docker/pkg/stringid/README.md create mode 100644 Godeps/_workspace/src/github.com/docker/docker/pkg/stringid/stringid.go create mode 100644 Godeps/_workspace/src/github.com/docker/docker/pkg/stringid/stringid_test.go create mode 100644 Godeps/_workspace/src/github.com/docker/libkv/.travis.yml create mode 100644 Godeps/_workspace/src/github.com/docker/libkv/LICENSE create mode 100644 Godeps/_workspace/src/github.com/docker/libkv/README.md create mode 100644 Godeps/_workspace/src/github.com/docker/libkv/libkv.go create mode 100644 Godeps/_workspace/src/github.com/docker/libkv/libkv_test.go create mode 100644 Godeps/_workspace/src/github.com/docker/libkv/script/.validate create mode 100644 Godeps/_workspace/src/github.com/docker/libkv/script/coverage create mode 100644 Godeps/_workspace/src/github.com/docker/libkv/script/travis_consul.sh create mode 100644 Godeps/_workspace/src/github.com/docker/libkv/script/travis_etcd.sh create mode 100644 Godeps/_workspace/src/github.com/docker/libkv/script/travis_zk.sh create mode 100644 Godeps/_workspace/src/github.com/docker/libkv/script/validate-gofmt create mode 100644 Godeps/_workspace/src/github.com/docker/libkv/store/consul/consul.go create mode 100644 Godeps/_workspace/src/github.com/docker/libkv/store/consul/consul_test.go create mode 100644 Godeps/_workspace/src/github.com/docker/libkv/store/etcd/etcd.go create mode 100644 Godeps/_workspace/src/github.com/docker/libkv/store/etcd/etcd_test.go create mode 100644 Godeps/_workspace/src/github.com/docker/libkv/store/helpers.go create mode 100644 Godeps/_workspace/src/github.com/docker/libkv/store/mock/mock.go create mode 100644 Godeps/_workspace/src/github.com/docker/libkv/store/store.go create mode 100644 Godeps/_workspace/src/github.com/docker/libkv/store/zookeeper/zookeeper.go create mode 100644 Godeps/_workspace/src/github.com/docker/libkv/store/zookeeper/zookeeper_test.go create mode 100644 Godeps/_workspace/src/github.com/docker/libkv/testutils/utils.go create mode 100644 Godeps/_workspace/src/github.com/docker/libnetwork/.gitignore create mode 100644 Godeps/_workspace/src/github.com/docker/libnetwork/Godeps/Godeps.json create mode 100644 Godeps/_workspace/src/github.com/docker/libnetwork/Godeps/Readme create mode 100644 Godeps/_workspace/src/github.com/docker/libnetwork/LICENSE create mode 100644 Godeps/_workspace/src/github.com/docker/libnetwork/MAINTAINERS create mode 100644 Godeps/_workspace/src/github.com/docker/libnetwork/Makefile create mode 100644 Godeps/_workspace/src/github.com/docker/libnetwork/README.md create mode 100644 Godeps/_workspace/src/github.com/docker/libnetwork/ROADMAP.md create mode 100644 Godeps/_workspace/src/github.com/docker/libnetwork/api/api.go create mode 100644 Godeps/_workspace/src/github.com/docker/libnetwork/api/api_test.go create mode 100644 Godeps/_workspace/src/github.com/docker/libnetwork/api/types.go create mode 100644 Godeps/_workspace/src/github.com/docker/libnetwork/bitseq/sequence.go create mode 100644 Godeps/_workspace/src/github.com/docker/libnetwork/bitseq/sequence_test.go create mode 100644 Godeps/_workspace/src/github.com/docker/libnetwork/bitseq/store.go create mode 100644 Godeps/_workspace/src/github.com/docker/libnetwork/circle.yml create mode 100644 Godeps/_workspace/src/github.com/docker/libnetwork/client/client.go create mode 100644 Godeps/_workspace/src/github.com/docker/libnetwork/client/client_service_test.go create mode 100644 Godeps/_workspace/src/github.com/docker/libnetwork/client/client_test.go create mode 100644 Godeps/_workspace/src/github.com/docker/libnetwork/client/network.go create mode 100644 Godeps/_workspace/src/github.com/docker/libnetwork/client/service.go create mode 100644 Godeps/_workspace/src/github.com/docker/libnetwork/client/types.go create mode 100644 Godeps/_workspace/src/github.com/docker/libnetwork/cmd/dnet/dnet.go create mode 100644 Godeps/_workspace/src/github.com/docker/libnetwork/cmd/dnet/dnet_test.go create mode 100644 Godeps/_workspace/src/github.com/docker/libnetwork/cmd/dnet/flags.go create mode 100644 Godeps/_workspace/src/github.com/docker/libnetwork/cmd/dnet/libnetwork.toml create mode 100644 Godeps/_workspace/src/github.com/docker/libnetwork/cmd/ovrouter/ovrouter.go create mode 100644 Godeps/_workspace/src/github.com/docker/libnetwork/cmd/readme_test/readme.go create mode 100644 Godeps/_workspace/src/github.com/docker/libnetwork/cmd/test/libnetwork.toml create mode 100644 Godeps/_workspace/src/github.com/docker/libnetwork/cmd/test/main.go create mode 100644 Godeps/_workspace/src/github.com/docker/libnetwork/config/config.go create mode 100644 Godeps/_workspace/src/github.com/docker/libnetwork/config/config_test.go create mode 100644 Godeps/_workspace/src/github.com/docker/libnetwork/config/libnetwork.toml create mode 100644 Godeps/_workspace/src/github.com/docker/libnetwork/controller.go create mode 100644 Godeps/_workspace/src/github.com/docker/libnetwork/datastore/datastore.go create mode 100644 Godeps/_workspace/src/github.com/docker/libnetwork/datastore/datastore_test.go create mode 100644 Godeps/_workspace/src/github.com/docker/libnetwork/datastore/mock_store.go create mode 100644 Godeps/_workspace/src/github.com/docker/libnetwork/docs/Vagrantfile create mode 100644 Godeps/_workspace/src/github.com/docker/libnetwork/docs/bridge.md create mode 100644 Godeps/_workspace/src/github.com/docker/libnetwork/docs/design.md create mode 100644 Godeps/_workspace/src/github.com/docker/libnetwork/docs/legacy.md create mode 100644 Godeps/_workspace/src/github.com/docker/libnetwork/docs/overlay.md create mode 100644 Godeps/_workspace/src/github.com/docker/libnetwork/docs/remote.md create mode 100644 Godeps/_workspace/src/github.com/docker/libnetwork/docs/vagrant-systemd/docker.service create mode 100644 Godeps/_workspace/src/github.com/docker/libnetwork/docs/vagrant.md create mode 100644 Godeps/_workspace/src/github.com/docker/libnetwork/driverapi/driverapi.go create mode 100644 Godeps/_workspace/src/github.com/docker/libnetwork/driverapi/errors.go create mode 100644 Godeps/_workspace/src/github.com/docker/libnetwork/drivers/bridge/bridge.go create mode 100644 Godeps/_workspace/src/github.com/docker/libnetwork/drivers/bridge/bridge_test.go create mode 100644 Godeps/_workspace/src/github.com/docker/libnetwork/drivers/bridge/errors.go create mode 100644 Godeps/_workspace/src/github.com/docker/libnetwork/drivers/bridge/interface.go create mode 100644 Godeps/_workspace/src/github.com/docker/libnetwork/drivers/bridge/interface_test.go create mode 100644 Godeps/_workspace/src/github.com/docker/libnetwork/drivers/bridge/link.go create mode 100644 Godeps/_workspace/src/github.com/docker/libnetwork/drivers/bridge/link_test.go create mode 100644 Godeps/_workspace/src/github.com/docker/libnetwork/drivers/bridge/netlink_deprecated_linux.go create mode 100644 Godeps/_workspace/src/github.com/docker/libnetwork/drivers/bridge/netlink_deprecated_linux_armppc64.go create mode 100644 Godeps/_workspace/src/github.com/docker/libnetwork/drivers/bridge/netlink_deprecated_linux_notarm.go create mode 100644 Godeps/_workspace/src/github.com/docker/libnetwork/drivers/bridge/netlink_deprecated_unsupported.go create mode 100644 Godeps/_workspace/src/github.com/docker/libnetwork/drivers/bridge/network_test.go create mode 100644 Godeps/_workspace/src/github.com/docker/libnetwork/drivers/bridge/port_mapping.go create mode 100644 Godeps/_workspace/src/github.com/docker/libnetwork/drivers/bridge/port_mapping_test.go create mode 100644 Godeps/_workspace/src/github.com/docker/libnetwork/drivers/bridge/resolvconf.go create mode 100644 Godeps/_workspace/src/github.com/docker/libnetwork/drivers/bridge/resolvconf_test.go create mode 100644 Godeps/_workspace/src/github.com/docker/libnetwork/drivers/bridge/setup.go create mode 100644 Godeps/_workspace/src/github.com/docker/libnetwork/drivers/bridge/setup_bridgenetfiltering.go create mode 100644 Godeps/_workspace/src/github.com/docker/libnetwork/drivers/bridge/setup_bridgenetfiltering_test.go create mode 100644 Godeps/_workspace/src/github.com/docker/libnetwork/drivers/bridge/setup_device.go create mode 100644 Godeps/_workspace/src/github.com/docker/libnetwork/drivers/bridge/setup_device_test.go create mode 100644 Godeps/_workspace/src/github.com/docker/libnetwork/drivers/bridge/setup_firewalld.go create mode 100644 Godeps/_workspace/src/github.com/docker/libnetwork/drivers/bridge/setup_fixedcidrv4.go create mode 100644 Godeps/_workspace/src/github.com/docker/libnetwork/drivers/bridge/setup_fixedcidrv4_test.go create mode 100644 Godeps/_workspace/src/github.com/docker/libnetwork/drivers/bridge/setup_fixedcidrv6.go create mode 100644 Godeps/_workspace/src/github.com/docker/libnetwork/drivers/bridge/setup_fixedcidrv6_test.go create mode 100644 Godeps/_workspace/src/github.com/docker/libnetwork/drivers/bridge/setup_ip_forwarding.go create mode 100644 Godeps/_workspace/src/github.com/docker/libnetwork/drivers/bridge/setup_ip_forwarding_test.go create mode 100644 Godeps/_workspace/src/github.com/docker/libnetwork/drivers/bridge/setup_ip_tables.go create mode 100644 Godeps/_workspace/src/github.com/docker/libnetwork/drivers/bridge/setup_ip_tables_test.go create mode 100644 Godeps/_workspace/src/github.com/docker/libnetwork/drivers/bridge/setup_ipv4.go create mode 100644 Godeps/_workspace/src/github.com/docker/libnetwork/drivers/bridge/setup_ipv4_test.go create mode 100644 Godeps/_workspace/src/github.com/docker/libnetwork/drivers/bridge/setup_ipv6.go create mode 100644 Godeps/_workspace/src/github.com/docker/libnetwork/drivers/bridge/setup_ipv6_test.go create mode 100644 Godeps/_workspace/src/github.com/docker/libnetwork/drivers/bridge/setup_verify.go create mode 100644 Godeps/_workspace/src/github.com/docker/libnetwork/drivers/bridge/setup_verify_test.go create mode 100644 Godeps/_workspace/src/github.com/docker/libnetwork/drivers/host/host.go create mode 100644 Godeps/_workspace/src/github.com/docker/libnetwork/drivers/host/host_test.go create mode 100644 Godeps/_workspace/src/github.com/docker/libnetwork/drivers/null/null.go create mode 100644 Godeps/_workspace/src/github.com/docker/libnetwork/drivers/null/null_test.go create mode 100644 Godeps/_workspace/src/github.com/docker/libnetwork/drivers/overlay/joinleave.go create mode 100644 Godeps/_workspace/src/github.com/docker/libnetwork/drivers/overlay/ov_endpoint.go create mode 100644 Godeps/_workspace/src/github.com/docker/libnetwork/drivers/overlay/ov_network.go create mode 100644 Godeps/_workspace/src/github.com/docker/libnetwork/drivers/overlay/ov_serf.go create mode 100644 Godeps/_workspace/src/github.com/docker/libnetwork/drivers/overlay/ov_utils.go create mode 100644 Godeps/_workspace/src/github.com/docker/libnetwork/drivers/overlay/overlay.go create mode 100644 Godeps/_workspace/src/github.com/docker/libnetwork/drivers/overlay/overlay_test.go create mode 100644 Godeps/_workspace/src/github.com/docker/libnetwork/drivers/overlay/peerdb.go create mode 100644 Godeps/_workspace/src/github.com/docker/libnetwork/drivers/remote/driver.go create mode 100644 Godeps/_workspace/src/github.com/docker/libnetwork/drivers/remote/driver_test.go create mode 100644 Godeps/_workspace/src/github.com/docker/libnetwork/drivers/windows/windows.go create mode 100644 Godeps/_workspace/src/github.com/docker/libnetwork/drivers_freebsd.go create mode 100644 Godeps/_workspace/src/github.com/docker/libnetwork/drivers_linux.go create mode 100644 Godeps/_workspace/src/github.com/docker/libnetwork/drivers_windows.go create mode 100644 Godeps/_workspace/src/github.com/docker/libnetwork/endpoint.go create mode 100644 Godeps/_workspace/src/github.com/docker/libnetwork/endpoint_info.go create mode 100644 Godeps/_workspace/src/github.com/docker/libnetwork/error.go create mode 100644 Godeps/_workspace/src/github.com/docker/libnetwork/errors_test.go create mode 100644 Godeps/_workspace/src/github.com/docker/libnetwork/etchosts/etchosts.go create mode 100644 Godeps/_workspace/src/github.com/docker/libnetwork/etchosts/etchosts_test.go create mode 100644 Godeps/_workspace/src/github.com/docker/libnetwork/hostdiscovery/hostdiscovery.go create mode 100644 Godeps/_workspace/src/github.com/docker/libnetwork/hostdiscovery/hostdiscovery_api.go create mode 100644 Godeps/_workspace/src/github.com/docker/libnetwork/hostdiscovery/hostdiscovery_disabled.go create mode 100644 Godeps/_workspace/src/github.com/docker/libnetwork/hostdiscovery/hostdiscovery_test.go create mode 100644 Godeps/_workspace/src/github.com/docker/libnetwork/hostdiscovery/libnetwork.toml create mode 100644 Godeps/_workspace/src/github.com/docker/libnetwork/idm/idm.go create mode 100644 Godeps/_workspace/src/github.com/docker/libnetwork/idm/idm_test.go create mode 100644 Godeps/_workspace/src/github.com/docker/libnetwork/ipallocator/allocator.go create mode 100644 Godeps/_workspace/src/github.com/docker/libnetwork/ipallocator/allocator_test.go create mode 100644 Godeps/_workspace/src/github.com/docker/libnetwork/ipam/allocator.go create mode 100644 Godeps/_workspace/src/github.com/docker/libnetwork/ipam/allocator_test.go create mode 100644 Godeps/_workspace/src/github.com/docker/libnetwork/ipam/contract.go create mode 100644 Godeps/_workspace/src/github.com/docker/libnetwork/ipam/store.go create mode 100644 Godeps/_workspace/src/github.com/docker/libnetwork/iptables/firewalld.go create mode 100644 Godeps/_workspace/src/github.com/docker/libnetwork/iptables/firewalld_test.go create mode 100644 Godeps/_workspace/src/github.com/docker/libnetwork/iptables/iptables.go create mode 100644 Godeps/_workspace/src/github.com/docker/libnetwork/iptables/iptables_test.go create mode 100644 Godeps/_workspace/src/github.com/docker/libnetwork/libnetwork_internal_test.go create mode 100644 Godeps/_workspace/src/github.com/docker/libnetwork/libnetwork_test.go create mode 100644 Godeps/_workspace/src/github.com/docker/libnetwork/netlabel/labels.go create mode 100644 Godeps/_workspace/src/github.com/docker/libnetwork/netutils/test_utils.go create mode 100644 Godeps/_workspace/src/github.com/docker/libnetwork/netutils/utils.go create mode 100644 Godeps/_workspace/src/github.com/docker/libnetwork/netutils/utils_test.go create mode 100644 Godeps/_workspace/src/github.com/docker/libnetwork/network.go create mode 100644 Godeps/_workspace/src/github.com/docker/libnetwork/options/options.go create mode 100644 Godeps/_workspace/src/github.com/docker/libnetwork/options/options_test.go create mode 100644 Godeps/_workspace/src/github.com/docker/libnetwork/portallocator/portallocator.go create mode 100644 Godeps/_workspace/src/github.com/docker/libnetwork/portallocator/portallocator_test.go create mode 100644 Godeps/_workspace/src/github.com/docker/libnetwork/portmapper/mapper.go create mode 100644 Godeps/_workspace/src/github.com/docker/libnetwork/portmapper/mapper_test.go create mode 100644 Godeps/_workspace/src/github.com/docker/libnetwork/portmapper/mock_proxy.go create mode 100644 Godeps/_workspace/src/github.com/docker/libnetwork/portmapper/proxy.go create mode 100644 Godeps/_workspace/src/github.com/docker/libnetwork/resolvconf/README.md create mode 100644 Godeps/_workspace/src/github.com/docker/libnetwork/resolvconf/dns/resolvconf.go create mode 100644 Godeps/_workspace/src/github.com/docker/libnetwork/resolvconf/resolvconf.go create mode 100644 Godeps/_workspace/src/github.com/docker/libnetwork/resolvconf/resolvconf_test.go create mode 100644 Godeps/_workspace/src/github.com/docker/libnetwork/sandbox/interface_freebsd.go create mode 100644 Godeps/_workspace/src/github.com/docker/libnetwork/sandbox/interface_linux.go create mode 100644 Godeps/_workspace/src/github.com/docker/libnetwork/sandbox/interface_windows.go create mode 100644 Godeps/_workspace/src/github.com/docker/libnetwork/sandbox/namespace_linux.go create mode 100644 Godeps/_workspace/src/github.com/docker/libnetwork/sandbox/namespace_unsupported.go create mode 100644 Godeps/_workspace/src/github.com/docker/libnetwork/sandbox/namespace_windows.go create mode 100644 Godeps/_workspace/src/github.com/docker/libnetwork/sandbox/neigh_freebsd.go create mode 100644 Godeps/_workspace/src/github.com/docker/libnetwork/sandbox/neigh_linux.go create mode 100644 Godeps/_workspace/src/github.com/docker/libnetwork/sandbox/neigh_windows.go create mode 100644 Godeps/_workspace/src/github.com/docker/libnetwork/sandbox/options_linux.go create mode 100644 Godeps/_workspace/src/github.com/docker/libnetwork/sandbox/route_linux.go create mode 100644 Godeps/_workspace/src/github.com/docker/libnetwork/sandbox/sandbox.go create mode 100644 Godeps/_workspace/src/github.com/docker/libnetwork/sandbox/sandbox_freebsd.go create mode 100644 Godeps/_workspace/src/github.com/docker/libnetwork/sandbox/sandbox_linux_test.go create mode 100644 Godeps/_workspace/src/github.com/docker/libnetwork/sandbox/sandbox_test.go create mode 100644 Godeps/_workspace/src/github.com/docker/libnetwork/sandbox/sandbox_unsupported.go create mode 100644 Godeps/_workspace/src/github.com/docker/libnetwork/sandbox/sandbox_unsupported_test.go create mode 100644 Godeps/_workspace/src/github.com/docker/libnetwork/sandboxdata.go create mode 100644 Godeps/_workspace/src/github.com/docker/libnetwork/sandboxdata_test.go create mode 100644 Godeps/_workspace/src/github.com/docker/libnetwork/store.go create mode 100644 Godeps/_workspace/src/github.com/docker/libnetwork/test/integration/README.md create mode 100644 Godeps/_workspace/src/github.com/docker/libnetwork/test/integration/daemon-configs.bats create mode 100644 Godeps/_workspace/src/github.com/docker/libnetwork/test/integration/daemon.cfg create mode 100644 Godeps/_workspace/src/github.com/docker/libnetwork/test/integration/helpers.bash create mode 100644 Godeps/_workspace/src/github.com/docker/libnetwork/types/types.go create mode 100644 Godeps/_workspace/src/github.com/docker/libnetwork/types/types_test.go create mode 100644 Godeps/_workspace/src/github.com/godbus/dbus/CONTRIBUTING.md create mode 100644 Godeps/_workspace/src/github.com/godbus/dbus/LICENSE create mode 100644 Godeps/_workspace/src/github.com/godbus/dbus/MAINTAINERS create mode 100644 Godeps/_workspace/src/github.com/godbus/dbus/README.markdown create mode 100644 Godeps/_workspace/src/github.com/godbus/dbus/auth.go create mode 100644 Godeps/_workspace/src/github.com/godbus/dbus/auth_external.go create mode 100644 Godeps/_workspace/src/github.com/godbus/dbus/auth_sha1.go create mode 100644 Godeps/_workspace/src/github.com/godbus/dbus/call.go create mode 100644 Godeps/_workspace/src/github.com/godbus/dbus/conn.go create mode 100644 Godeps/_workspace/src/github.com/godbus/dbus/conn_darwin.go create mode 100644 Godeps/_workspace/src/github.com/godbus/dbus/conn_other.go create mode 100644 Godeps/_workspace/src/github.com/godbus/dbus/conn_test.go create mode 100644 Godeps/_workspace/src/github.com/godbus/dbus/dbus.go create mode 100644 Godeps/_workspace/src/github.com/godbus/dbus/decoder.go create mode 100644 Godeps/_workspace/src/github.com/godbus/dbus/doc.go create mode 100644 Godeps/_workspace/src/github.com/godbus/dbus/encoder.go create mode 100644 Godeps/_workspace/src/github.com/godbus/dbus/encoder_test.go create mode 100644 Godeps/_workspace/src/github.com/godbus/dbus/examples_test.go create mode 100644 Godeps/_workspace/src/github.com/godbus/dbus/export.go create mode 100644 Godeps/_workspace/src/github.com/godbus/dbus/homedir.go create mode 100644 Godeps/_workspace/src/github.com/godbus/dbus/homedir_dynamic.go create mode 100644 Godeps/_workspace/src/github.com/godbus/dbus/homedir_static.go create mode 100644 Godeps/_workspace/src/github.com/godbus/dbus/introspect/call.go create mode 100644 Godeps/_workspace/src/github.com/godbus/dbus/introspect/introspect.go create mode 100644 Godeps/_workspace/src/github.com/godbus/dbus/introspect/introspectable.go create mode 100644 Godeps/_workspace/src/github.com/godbus/dbus/message.go create mode 100644 Godeps/_workspace/src/github.com/godbus/dbus/prop/prop.go create mode 100644 Godeps/_workspace/src/github.com/godbus/dbus/proto_test.go create mode 100644 Godeps/_workspace/src/github.com/godbus/dbus/sig.go create mode 100644 Godeps/_workspace/src/github.com/godbus/dbus/sig_test.go create mode 100644 Godeps/_workspace/src/github.com/godbus/dbus/transport_darwin.go create mode 100644 Godeps/_workspace/src/github.com/godbus/dbus/transport_generic.go create mode 100644 Godeps/_workspace/src/github.com/godbus/dbus/transport_unix.go create mode 100644 Godeps/_workspace/src/github.com/godbus/dbus/transport_unix_test.go create mode 100644 Godeps/_workspace/src/github.com/godbus/dbus/transport_unixcred_dragonfly.go create mode 100644 Godeps/_workspace/src/github.com/godbus/dbus/transport_unixcred_linux.go create mode 100644 Godeps/_workspace/src/github.com/godbus/dbus/variant.go create mode 100644 Godeps/_workspace/src/github.com/godbus/dbus/variant_lexer.go create mode 100644 Godeps/_workspace/src/github.com/godbus/dbus/variant_parser.go create mode 100644 Godeps/_workspace/src/github.com/godbus/dbus/variant_test.go create mode 100644 Godeps/_workspace/src/github.com/hashicorp/go-msgpack/codec/0doc.go create mode 100644 Godeps/_workspace/src/github.com/hashicorp/go-msgpack/codec/README.md create mode 100644 Godeps/_workspace/src/github.com/hashicorp/go-msgpack/codec/bench_test.go create mode 100644 Godeps/_workspace/src/github.com/hashicorp/go-msgpack/codec/binc.go create mode 100644 Godeps/_workspace/src/github.com/hashicorp/go-msgpack/codec/codecs_test.go create mode 100644 Godeps/_workspace/src/github.com/hashicorp/go-msgpack/codec/decode.go create mode 100644 Godeps/_workspace/src/github.com/hashicorp/go-msgpack/codec/encode.go create mode 100644 Godeps/_workspace/src/github.com/hashicorp/go-msgpack/codec/ext_dep_test.go create mode 100644 Godeps/_workspace/src/github.com/hashicorp/go-msgpack/codec/helper.go create mode 100644 Godeps/_workspace/src/github.com/hashicorp/go-msgpack/codec/helper_internal.go create mode 100644 Godeps/_workspace/src/github.com/hashicorp/go-msgpack/codec/msgpack.go create mode 100644 Godeps/_workspace/src/github.com/hashicorp/go-msgpack/codec/msgpack_test.py create mode 100644 Godeps/_workspace/src/github.com/hashicorp/go-msgpack/codec/rpc.go create mode 100644 Godeps/_workspace/src/github.com/hashicorp/go-msgpack/codec/simple.go create mode 100644 Godeps/_workspace/src/github.com/hashicorp/go-msgpack/codec/time.go create mode 100644 Godeps/_workspace/src/github.com/hashicorp/go-msgpack/codec/z_helper_test.go create mode 100644 Godeps/_workspace/src/github.com/hashicorp/memberlist/.gitignore create mode 100644 Godeps/_workspace/src/github.com/hashicorp/memberlist/LICENSE create mode 100644 Godeps/_workspace/src/github.com/hashicorp/memberlist/Makefile create mode 100644 Godeps/_workspace/src/github.com/hashicorp/memberlist/README.md create mode 100644 Godeps/_workspace/src/github.com/hashicorp/memberlist/broadcast.go create mode 100644 Godeps/_workspace/src/github.com/hashicorp/memberlist/broadcast_test.go create mode 100644 Godeps/_workspace/src/github.com/hashicorp/memberlist/config.go create mode 100644 Godeps/_workspace/src/github.com/hashicorp/memberlist/conflict_delegate.go create mode 100644 Godeps/_workspace/src/github.com/hashicorp/memberlist/delegate.go create mode 100644 Godeps/_workspace/src/github.com/hashicorp/memberlist/event_delegate.go create mode 100644 Godeps/_workspace/src/github.com/hashicorp/memberlist/integ_test.go create mode 100644 Godeps/_workspace/src/github.com/hashicorp/memberlist/keyring.go create mode 100644 Godeps/_workspace/src/github.com/hashicorp/memberlist/keyring_test.go create mode 100644 Godeps/_workspace/src/github.com/hashicorp/memberlist/memberlist.go create mode 100644 Godeps/_workspace/src/github.com/hashicorp/memberlist/memberlist_test.go create mode 100644 Godeps/_workspace/src/github.com/hashicorp/memberlist/merge_delegate.go create mode 100644 Godeps/_workspace/src/github.com/hashicorp/memberlist/net.go create mode 100644 Godeps/_workspace/src/github.com/hashicorp/memberlist/net_test.go create mode 100644 Godeps/_workspace/src/github.com/hashicorp/memberlist/queue.go create mode 100644 Godeps/_workspace/src/github.com/hashicorp/memberlist/queue_test.go create mode 100644 Godeps/_workspace/src/github.com/hashicorp/memberlist/security.go create mode 100644 Godeps/_workspace/src/github.com/hashicorp/memberlist/security_test.go create mode 100644 Godeps/_workspace/src/github.com/hashicorp/memberlist/state.go create mode 100644 Godeps/_workspace/src/github.com/hashicorp/memberlist/state_test.go create mode 100644 Godeps/_workspace/src/github.com/hashicorp/memberlist/test/setup_subnet.sh create mode 100644 Godeps/_workspace/src/github.com/hashicorp/memberlist/todo.md create mode 100644 Godeps/_workspace/src/github.com/hashicorp/memberlist/util.go create mode 100644 Godeps/_workspace/src/github.com/hashicorp/memberlist/util_test.go create mode 100644 Godeps/_workspace/src/github.com/hashicorp/serf/serf/broadcast.go create mode 100644 Godeps/_workspace/src/github.com/hashicorp/serf/serf/broadcast_test.go create mode 100644 Godeps/_workspace/src/github.com/hashicorp/serf/serf/coalesce.go create mode 100644 Godeps/_workspace/src/github.com/hashicorp/serf/serf/coalesce_member.go create mode 100644 Godeps/_workspace/src/github.com/hashicorp/serf/serf/coalesce_member_test.go create mode 100644 Godeps/_workspace/src/github.com/hashicorp/serf/serf/coalesce_test.go create mode 100644 Godeps/_workspace/src/github.com/hashicorp/serf/serf/coalesce_user.go create mode 100644 Godeps/_workspace/src/github.com/hashicorp/serf/serf/coalesce_user_test.go create mode 100644 Godeps/_workspace/src/github.com/hashicorp/serf/serf/config.go create mode 100644 Godeps/_workspace/src/github.com/hashicorp/serf/serf/config_test.go create mode 100644 Godeps/_workspace/src/github.com/hashicorp/serf/serf/conflict_delegate.go create mode 100644 Godeps/_workspace/src/github.com/hashicorp/serf/serf/delegate.go create mode 100644 Godeps/_workspace/src/github.com/hashicorp/serf/serf/delegate_test.go create mode 100644 Godeps/_workspace/src/github.com/hashicorp/serf/serf/event.go create mode 100644 Godeps/_workspace/src/github.com/hashicorp/serf/serf/event_delegate.go create mode 100644 Godeps/_workspace/src/github.com/hashicorp/serf/serf/event_test.go create mode 100644 Godeps/_workspace/src/github.com/hashicorp/serf/serf/internal_query.go create mode 100644 Godeps/_workspace/src/github.com/hashicorp/serf/serf/internal_query_test.go create mode 100644 Godeps/_workspace/src/github.com/hashicorp/serf/serf/keymanager.go create mode 100644 Godeps/_workspace/src/github.com/hashicorp/serf/serf/keymanager_test.go create mode 100644 Godeps/_workspace/src/github.com/hashicorp/serf/serf/lamport.go create mode 100644 Godeps/_workspace/src/github.com/hashicorp/serf/serf/lamport_test.go create mode 100644 Godeps/_workspace/src/github.com/hashicorp/serf/serf/merge_delegate.go create mode 100644 Godeps/_workspace/src/github.com/hashicorp/serf/serf/messages.go create mode 100644 Godeps/_workspace/src/github.com/hashicorp/serf/serf/messages_test.go create mode 100644 Godeps/_workspace/src/github.com/hashicorp/serf/serf/query.go create mode 100644 Godeps/_workspace/src/github.com/hashicorp/serf/serf/query_test.go create mode 100644 Godeps/_workspace/src/github.com/hashicorp/serf/serf/serf.go create mode 100644 Godeps/_workspace/src/github.com/hashicorp/serf/serf/serf_internals_test.go create mode 100644 Godeps/_workspace/src/github.com/hashicorp/serf/serf/serf_test.go create mode 100644 Godeps/_workspace/src/github.com/hashicorp/serf/serf/snapshot.go create mode 100644 Godeps/_workspace/src/github.com/hashicorp/serf/serf/snapshot_test.go create mode 100644 Godeps/_workspace/src/github.com/samuel/go-zookeeper/zk/cluster_test.go create mode 100644 Godeps/_workspace/src/github.com/samuel/go-zookeeper/zk/conn.go create mode 100644 Godeps/_workspace/src/github.com/samuel/go-zookeeper/zk/constants.go create mode 100644 Godeps/_workspace/src/github.com/samuel/go-zookeeper/zk/constants_test.go create mode 100644 Godeps/_workspace/src/github.com/samuel/go-zookeeper/zk/flw.go create mode 100644 Godeps/_workspace/src/github.com/samuel/go-zookeeper/zk/flw_test.go create mode 100644 Godeps/_workspace/src/github.com/samuel/go-zookeeper/zk/lock.go create mode 100644 Godeps/_workspace/src/github.com/samuel/go-zookeeper/zk/lock_test.go create mode 100644 Godeps/_workspace/src/github.com/samuel/go-zookeeper/zk/server_help.go create mode 100644 Godeps/_workspace/src/github.com/samuel/go-zookeeper/zk/server_java.go create mode 100644 Godeps/_workspace/src/github.com/samuel/go-zookeeper/zk/structs.go create mode 100644 Godeps/_workspace/src/github.com/samuel/go-zookeeper/zk/structs_test.go create mode 100644 Godeps/_workspace/src/github.com/samuel/go-zookeeper/zk/tracer.go create mode 100644 Godeps/_workspace/src/github.com/samuel/go-zookeeper/zk/util.go create mode 100644 Godeps/_workspace/src/github.com/samuel/go-zookeeper/zk/util_test.go create mode 100644 Godeps/_workspace/src/github.com/samuel/go-zookeeper/zk/zk_test.go create mode 100644 Godeps/_workspace/src/github.com/vishvananda/netns/LICENSE create mode 100644 Godeps/_workspace/src/github.com/vishvananda/netns/README.md create mode 100644 Godeps/_workspace/src/github.com/vishvananda/netns/netns.go create mode 100644 Godeps/_workspace/src/github.com/vishvananda/netns/netns_linux.go create mode 100644 Godeps/_workspace/src/github.com/vishvananda/netns/netns_linux_386.go create mode 100644 Godeps/_workspace/src/github.com/vishvananda/netns/netns_linux_amd64.go create mode 100644 Godeps/_workspace/src/github.com/vishvananda/netns/netns_linux_arm.go create mode 100644 Godeps/_workspace/src/github.com/vishvananda/netns/netns_linux_arm64.go create mode 100644 Godeps/_workspace/src/github.com/vishvananda/netns/netns_linux_ppc64le.go create mode 100644 Godeps/_workspace/src/github.com/vishvananda/netns/netns_test.go create mode 100644 Godeps/_workspace/src/github.com/vishvananda/netns/netns_unspecified.go diff --git a/Godeps/Godeps.json b/Godeps/Godeps.json index 82aa949f3..3110537ef 100644 --- a/Godeps/Godeps.json +++ b/Godeps/Godeps.json @@ -5,11 +5,20 @@ "./..." ], "Deps": [ + { + "ImportPath": "github.com/BurntSushi/toml", + "Comment": "v0.1.0-16-gf706d00", + "Rev": "f706d00e3de6abe700c994cdd545a1a4915af060" + }, { "ImportPath": "github.com/Sirupsen/logrus", "Comment": "v0.8.2-10-g21d4508", "Rev": "21d4508646ae56d79244bd9046c1df63a5fa8c37" }, + { + "ImportPath": "github.com/armon/go-metrics", + "Rev": "eb0af217e5e9747e41dd5303755356b62d28e3ec" + }, { "ImportPath": "github.com/cenkalti/hub", "Comment": "v1.0.0-12-g7be60e1", @@ -33,6 +42,10 @@ "ImportPath": "github.com/contiv/objmodel/contivModel", "Rev": "dd43e14dc02b8e0d85ac8469d00beac5386af16d" }, + { + "ImportPath": "github.com/contiv/objmodel/objdb", + "Rev": "dd43e14dc02b8e0d85ac8469d00beac5386af16d" + }, { "ImportPath": "github.com/contiv/ofnet", "Rev": "2ed83a9c24b58c6bbdb807a771947958ad352bbd" @@ -47,21 +60,56 @@ "Comment": "v2.1.0-alpha.0-51-gb24dd8e", "Rev": "b24dd8e4e67d7203c5a68cd43de4f6ffcfff5e66" }, + { + "ImportPath": "github.com/coreos/go-etcd/etcd", + "Comment": "v2.0.0-7-g73a8ef7", + "Rev": "73a8ef737e8ea002281a28b4cb92a1de121ad4c6" + }, + { + "ImportPath": "github.com/docker/docker/pkg/ioutils", + "Comment": "v1.4.1-5404-gc471b7a", + "Rev": "c471b7aba50f5034bf44853b1dd3520bd02391ac" + }, { "ImportPath": "github.com/docker/docker/pkg/listenbuffer", "Comment": "v1.4.1-5404-gc471b7a", "Rev": "c471b7aba50f5034bf44853b1dd3520bd02391ac" }, + { + "ImportPath": "github.com/docker/docker/pkg/parsers/kernel", + "Comment": "v1.4.1-5404-gc471b7a", + "Rev": "c471b7aba50f5034bf44853b1dd3520bd02391ac" + }, { "ImportPath": "github.com/docker/docker/pkg/plugins", "Comment": "v1.4.1-5404-gc471b7a", "Rev": "c471b7aba50f5034bf44853b1dd3520bd02391ac" }, + { + "ImportPath": "github.com/docker/docker/pkg/proxy", + "Comment": "v1.4.1-5404-gc471b7a", + "Rev": "c471b7aba50f5034bf44853b1dd3520bd02391ac" + }, + { + "ImportPath": "github.com/docker/docker/pkg/random", + "Comment": "v1.4.1-5404-gc471b7a", + "Rev": "c471b7aba50f5034bf44853b1dd3520bd02391ac" + }, + { + "ImportPath": "github.com/docker/docker/pkg/reexec", + "Comment": "v1.4.1-5404-gc471b7a", + "Rev": "c471b7aba50f5034bf44853b1dd3520bd02391ac" + }, { "ImportPath": "github.com/docker/docker/pkg/sockets", "Comment": "v1.4.1-5404-gc471b7a", "Rev": "c471b7aba50f5034bf44853b1dd3520bd02391ac" }, + { + "ImportPath": "github.com/docker/docker/pkg/stringid", + "Comment": "v1.4.1-5404-gc471b7a", + "Rev": "c471b7aba50f5034bf44853b1dd3520bd02391ac" + }, { "ImportPath": "github.com/docker/docker/pkg/tlsconfig", "Comment": "v1.4.1-5404-gc471b7a", @@ -73,9 +121,18 @@ "Rev": "c471b7aba50f5034bf44853b1dd3520bd02391ac" }, { - "ImportPath": "github.com/docker/libnetwork/drivers/remote/api", - "Comment": "v0.2-335-g8280c18", - "Rev": "8280c188e2795f9327365e872db56956974ec7c6" + "ImportPath": "github.com/docker/libkv", + "Rev": "60c7c881345b3c67defc7f93a8297debf041d43c" + }, + { + "ImportPath": "github.com/docker/libnetwork", + "Comment": "v0.2-359-g0dca755", + "Rev": "0dca75517b2feb75890b20aaad2652f3a0d8ff10" + }, + { + "ImportPath": "github.com/godbus/dbus", + "Comment": "v2-3-g4160802", + "Rev": "41608027bdce7bfa8959d653a00b954591220e67" }, { "ImportPath": "github.com/gorilla/context", @@ -90,16 +147,24 @@ "Comment": "v0.5.2-30-g26d15e6", "Rev": "26d15e6f07af180e35bdd694d40ecf71d4929656" }, + { + "ImportPath": "github.com/hashicorp/go-msgpack/codec", + "Rev": "71c2886f5a673a35f909803f38ece5810165097b" + }, + { + "ImportPath": "github.com/hashicorp/memberlist", + "Rev": "9a1e242e454d2443df330bdd51a436d5a9058fc4" + }, + { + "ImportPath": "github.com/hashicorp/serf/serf", + "Comment": "v0.6.4", + "Rev": "7151adcef72687bf95f451a2e0ba15cb19412bf2" + }, { "ImportPath": "github.com/jainvipin/bitset", "Comment": "v1.0.0-12-g1f0c6de", "Rev": "1f0c6de81a62732e75f6c97f1e28e5a009b5df68" }, - { - "ImportPath": "github.com/mapuri/libnetwork/driverapi", - "Comment": "v0.1-34-gbf9450e", - "Rev": "bf9450eec0646abd7e71dc3eaff47a55f7a1f1df" - }, { "ImportPath": "github.com/opencontainers/runc/libcontainer/user", "Comment": "v0.0.3-14-g9be9157", @@ -109,6 +174,10 @@ "ImportPath": "github.com/samalba/dockerclient", "Rev": "4e6f4a21c07510dd6446d28053351a275591f08d" }, + { + "ImportPath": "github.com/samuel/go-zookeeper/zk", + "Rev": "d0e0d8e11f318e000a8cc434616d69e329edc374" + }, { "ImportPath": "github.com/shaleman/libOpenflow/common", "Rev": "50a0b02df3695fb6e892ad0f82b8a7c4c61fcf51" @@ -127,7 +196,11 @@ }, { "ImportPath": "github.com/vishvananda/netlink", - "Rev": "ae3e7dba57271b4e976c4f91637861ee477135e2" + "Rev": "4b5dce31de6d42af5bb9811c6d265472199e0fec" + }, + { + "ImportPath": "github.com/vishvananda/netns", + "Rev": "493029407eeb434d0c2d44e02ea072ff2488d322" }, { "ImportPath": "golang.org/x/crypto/ssh", diff --git a/Godeps/_workspace/src/github.com/BurntSushi/toml/.gitignore b/Godeps/_workspace/src/github.com/BurntSushi/toml/.gitignore new file mode 100644 index 000000000..0cd380037 --- /dev/null +++ b/Godeps/_workspace/src/github.com/BurntSushi/toml/.gitignore @@ -0,0 +1,5 @@ +TAGS +tags +.*.swp +tomlcheck/tomlcheck +toml.test diff --git a/Godeps/_workspace/src/github.com/BurntSushi/toml/.travis.yml b/Godeps/_workspace/src/github.com/BurntSushi/toml/.travis.yml new file mode 100644 index 000000000..43caf6d02 --- /dev/null +++ b/Godeps/_workspace/src/github.com/BurntSushi/toml/.travis.yml @@ -0,0 +1,12 @@ +language: go +go: + - 1.1 + - 1.2 + - tip +install: + - go install ./... + - go get github.com/BurntSushi/toml-test +script: + - export PATH="$PATH:$HOME/gopath/bin" + - make test + diff --git a/Godeps/_workspace/src/github.com/BurntSushi/toml/COMPATIBLE b/Godeps/_workspace/src/github.com/BurntSushi/toml/COMPATIBLE new file mode 100644 index 000000000..21e0938ca --- /dev/null +++ b/Godeps/_workspace/src/github.com/BurntSushi/toml/COMPATIBLE @@ -0,0 +1,3 @@ +Compatible with TOML version +[v0.2.0](https://github.com/mojombo/toml/blob/master/versions/toml-v0.2.0.md) + diff --git a/Godeps/_workspace/src/github.com/BurntSushi/toml/COPYING b/Godeps/_workspace/src/github.com/BurntSushi/toml/COPYING new file mode 100644 index 000000000..5a8e33254 --- /dev/null +++ b/Godeps/_workspace/src/github.com/BurntSushi/toml/COPYING @@ -0,0 +1,14 @@ + DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE + Version 2, December 2004 + + Copyright (C) 2004 Sam Hocevar + + Everyone is permitted to copy and distribute verbatim or modified + copies of this license document, and changing it is allowed as long + as the name is changed. + + DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. You just DO WHAT THE FUCK YOU WANT TO. + diff --git a/Godeps/_workspace/src/github.com/BurntSushi/toml/Makefile b/Godeps/_workspace/src/github.com/BurntSushi/toml/Makefile new file mode 100644 index 000000000..3600848d3 --- /dev/null +++ b/Godeps/_workspace/src/github.com/BurntSushi/toml/Makefile @@ -0,0 +1,19 @@ +install: + go install ./... + +test: install + go test -v + toml-test toml-test-decoder + toml-test -encoder toml-test-encoder + +fmt: + gofmt -w *.go */*.go + colcheck *.go */*.go + +tags: + find ./ -name '*.go' -print0 | xargs -0 gotags > TAGS + +push: + git push origin master + git push github master + diff --git a/Godeps/_workspace/src/github.com/BurntSushi/toml/README.md b/Godeps/_workspace/src/github.com/BurntSushi/toml/README.md new file mode 100644 index 000000000..e861c0ca7 --- /dev/null +++ b/Godeps/_workspace/src/github.com/BurntSushi/toml/README.md @@ -0,0 +1,220 @@ +## TOML parser and encoder for Go with reflection + +TOML stands for Tom's Obvious, Minimal Language. This Go package provides a +reflection interface similar to Go's standard library `json` and `xml` +packages. This package also supports the `encoding.TextUnmarshaler` and +`encoding.TextMarshaler` interfaces so that you can define custom data +representations. (There is an example of this below.) + +Spec: https://github.com/mojombo/toml + +Compatible with TOML version +[v0.2.0](https://github.com/mojombo/toml/blob/master/versions/toml-v0.2.0.md) + +Documentation: http://godoc.org/github.com/BurntSushi/toml + +Installation: + +```bash +go get github.com/BurntSushi/toml +``` + +Try the toml validator: + +```bash +go get github.com/BurntSushi/toml/cmd/tomlv +tomlv some-toml-file.toml +``` + +[![Build status](https://api.travis-ci.org/BurntSushi/toml.png)](https://travis-ci.org/BurntSushi/toml) + + +### Testing + +This package passes all tests in +[toml-test](https://github.com/BurntSushi/toml-test) for both the decoder +and the encoder. + +### Examples + +This package works similarly to how the Go standard library handles `XML` +and `JSON`. Namely, data is loaded into Go values via reflection. + +For the simplest example, consider some TOML file as just a list of keys +and values: + +```toml +Age = 25 +Cats = [ "Cauchy", "Plato" ] +Pi = 3.14 +Perfection = [ 6, 28, 496, 8128 ] +DOB = 1987-07-05T05:45:00Z +``` + +Which could be defined in Go as: + +```go +type Config struct { + Age int + Cats []string + Pi float64 + Perfection []int + DOB time.Time // requires `import time` +} +``` + +And then decoded with: + +```go +var conf Config +if _, err := toml.Decode(tomlData, &conf); err != nil { + // handle error +} +``` + +You can also use struct tags if your struct field name doesn't map to a TOML +key value directly: + +```toml +some_key_NAME = "wat" +``` + +```go +type TOML struct { + ObscureKey string `toml:"some_key_NAME"` +} +``` + +### Using the `encoding.TextUnmarshaler` interface + +Here's an example that automatically parses duration strings into +`time.Duration` values: + +```toml +[[song]] +name = "Thunder Road" +duration = "4m49s" + +[[song]] +name = "Stairway to Heaven" +duration = "8m03s" +``` + +Which can be decoded with: + +```go +type song struct { + Name string + Duration duration +} +type songs struct { + Song []song +} +var favorites songs +if _, err := toml.Decode(blob, &favorites); err != nil { + log.Fatal(err) +} + +for _, s := range favorites.Song { + fmt.Printf("%s (%s)\n", s.Name, s.Duration) +} +``` + +And you'll also need a `duration` type that satisfies the +`encoding.TextUnmarshaler` interface: + +```go +type duration struct { + time.Duration +} + +func (d *duration) UnmarshalText(text []byte) error { + var err error + d.Duration, err = time.ParseDuration(string(text)) + return err +} +``` + +### More complex usage + +Here's an example of how to load the example from the official spec page: + +```toml +# This is a TOML document. Boom. + +title = "TOML Example" + +[owner] +name = "Tom Preston-Werner" +organization = "GitHub" +bio = "GitHub Cofounder & CEO\nLikes tater tots and beer." +dob = 1979-05-27T07:32:00Z # First class dates? Why not? + +[database] +server = "192.168.1.1" +ports = [ 8001, 8001, 8002 ] +connection_max = 5000 +enabled = true + +[servers] + + # You can indent as you please. Tabs or spaces. TOML don't care. + [servers.alpha] + ip = "10.0.0.1" + dc = "eqdc10" + + [servers.beta] + ip = "10.0.0.2" + dc = "eqdc10" + +[clients] +data = [ ["gamma", "delta"], [1, 2] ] # just an update to make sure parsers support it + +# Line breaks are OK when inside arrays +hosts = [ + "alpha", + "omega" +] +``` + +And the corresponding Go types are: + +```go +type tomlConfig struct { + Title string + Owner ownerInfo + DB database `toml:"database"` + Servers map[string]server + Clients clients +} + +type ownerInfo struct { + Name string + Org string `toml:"organization"` + Bio string + DOB time.Time +} + +type database struct { + Server string + Ports []int + ConnMax int `toml:"connection_max"` + Enabled bool +} + +type server struct { + IP string + DC string +} + +type clients struct { + Data [][]interface{} + Hosts []string +} +``` + +Note that a case insensitive match will be tried if an exact match can't be +found. + +A working example of the above can be found in `_examples/example.{go,toml}`. + diff --git a/Godeps/_workspace/src/github.com/BurntSushi/toml/cmd/toml-test-decoder/COPYING b/Godeps/_workspace/src/github.com/BurntSushi/toml/cmd/toml-test-decoder/COPYING new file mode 100644 index 000000000..5a8e33254 --- /dev/null +++ b/Godeps/_workspace/src/github.com/BurntSushi/toml/cmd/toml-test-decoder/COPYING @@ -0,0 +1,14 @@ + DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE + Version 2, December 2004 + + Copyright (C) 2004 Sam Hocevar + + Everyone is permitted to copy and distribute verbatim or modified + copies of this license document, and changing it is allowed as long + as the name is changed. + + DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. You just DO WHAT THE FUCK YOU WANT TO. + diff --git a/Godeps/_workspace/src/github.com/BurntSushi/toml/cmd/toml-test-decoder/README.md b/Godeps/_workspace/src/github.com/BurntSushi/toml/cmd/toml-test-decoder/README.md new file mode 100644 index 000000000..24421eb70 --- /dev/null +++ b/Godeps/_workspace/src/github.com/BurntSushi/toml/cmd/toml-test-decoder/README.md @@ -0,0 +1,14 @@ +# Implements the TOML test suite interface + +This is an implementation of the interface expected by +[toml-test](https://github.com/BurntSushi/toml-test) for my +[toml parser written in Go](https://github.com/BurntSushi/toml). +In particular, it maps TOML data on `stdin` to a JSON format on `stdout`. + + +Compatible with TOML version +[v0.2.0](https://github.com/mojombo/toml/blob/master/versions/toml-v0.2.0.md) + +Compatible with `toml-test` version +[v0.2.0](https://github.com/BurntSushi/toml-test/tree/v0.2.0) + diff --git a/Godeps/_workspace/src/github.com/BurntSushi/toml/cmd/toml-test-decoder/main.go b/Godeps/_workspace/src/github.com/BurntSushi/toml/cmd/toml-test-decoder/main.go new file mode 100644 index 000000000..14e755700 --- /dev/null +++ b/Godeps/_workspace/src/github.com/BurntSushi/toml/cmd/toml-test-decoder/main.go @@ -0,0 +1,90 @@ +// Command toml-test-decoder satisfies the toml-test interface for testing +// TOML decoders. Namely, it accepts TOML on stdin and outputs JSON on stdout. +package main + +import ( + "encoding/json" + "flag" + "fmt" + "log" + "os" + "path" + "time" + + "github.com/BurntSushi/toml" +) + +func init() { + log.SetFlags(0) + + flag.Usage = usage + flag.Parse() +} + +func usage() { + log.Printf("Usage: %s < toml-file\n", path.Base(os.Args[0])) + flag.PrintDefaults() + + os.Exit(1) +} + +func main() { + if flag.NArg() != 0 { + flag.Usage() + } + + var tmp interface{} + if _, err := toml.DecodeReader(os.Stdin, &tmp); err != nil { + log.Fatalf("Error decoding TOML: %s", err) + } + + typedTmp := translate(tmp) + if err := json.NewEncoder(os.Stdout).Encode(typedTmp); err != nil { + log.Fatalf("Error encoding JSON: %s", err) + } +} + +func translate(tomlData interface{}) interface{} { + switch orig := tomlData.(type) { + case map[string]interface{}: + typed := make(map[string]interface{}, len(orig)) + for k, v := range orig { + typed[k] = translate(v) + } + return typed + case []map[string]interface{}: + typed := make([]map[string]interface{}, len(orig)) + for i, v := range orig { + typed[i] = translate(v).(map[string]interface{}) + } + return typed + case []interface{}: + typed := make([]interface{}, len(orig)) + for i, v := range orig { + typed[i] = translate(v) + } + + // We don't really need to tag arrays, but let's be future proof. + // (If TOML ever supports tuples, we'll need this.) + return tag("array", typed) + case time.Time: + return tag("datetime", orig.Format("2006-01-02T15:04:05Z")) + case bool: + return tag("bool", fmt.Sprintf("%v", orig)) + case int64: + return tag("integer", fmt.Sprintf("%d", orig)) + case float64: + return tag("float", fmt.Sprintf("%v", orig)) + case string: + return tag("string", orig) + } + + panic(fmt.Sprintf("Unknown type: %T", tomlData)) +} + +func tag(typeName string, data interface{}) map[string]interface{} { + return map[string]interface{}{ + "type": typeName, + "value": data, + } +} diff --git a/Godeps/_workspace/src/github.com/BurntSushi/toml/cmd/toml-test-encoder/COPYING b/Godeps/_workspace/src/github.com/BurntSushi/toml/cmd/toml-test-encoder/COPYING new file mode 100644 index 000000000..5a8e33254 --- /dev/null +++ b/Godeps/_workspace/src/github.com/BurntSushi/toml/cmd/toml-test-encoder/COPYING @@ -0,0 +1,14 @@ + DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE + Version 2, December 2004 + + Copyright (C) 2004 Sam Hocevar + + Everyone is permitted to copy and distribute verbatim or modified + copies of this license document, and changing it is allowed as long + as the name is changed. + + DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. You just DO WHAT THE FUCK YOU WANT TO. + diff --git a/Godeps/_workspace/src/github.com/BurntSushi/toml/cmd/toml-test-encoder/README.md b/Godeps/_workspace/src/github.com/BurntSushi/toml/cmd/toml-test-encoder/README.md new file mode 100644 index 000000000..45a603f29 --- /dev/null +++ b/Godeps/_workspace/src/github.com/BurntSushi/toml/cmd/toml-test-encoder/README.md @@ -0,0 +1,14 @@ +# Implements the TOML test suite interface for TOML encoders + +This is an implementation of the interface expected by +[toml-test](https://github.com/BurntSushi/toml-test) for the +[TOML encoder](https://github.com/BurntSushi/toml). +In particular, it maps JSON data on `stdin` to a TOML format on `stdout`. + + +Compatible with TOML version +[v0.2.0](https://github.com/mojombo/toml/blob/master/versions/toml-v0.2.0.md) + +Compatible with `toml-test` version +[v0.2.0](https://github.com/BurntSushi/toml-test/tree/v0.2.0) + diff --git a/Godeps/_workspace/src/github.com/BurntSushi/toml/cmd/toml-test-encoder/main.go b/Godeps/_workspace/src/github.com/BurntSushi/toml/cmd/toml-test-encoder/main.go new file mode 100644 index 000000000..092cc6844 --- /dev/null +++ b/Godeps/_workspace/src/github.com/BurntSushi/toml/cmd/toml-test-encoder/main.go @@ -0,0 +1,131 @@ +// Command toml-test-encoder satisfies the toml-test interface for testing +// TOML encoders. Namely, it accepts JSON on stdin and outputs TOML on stdout. +package main + +import ( + "encoding/json" + "flag" + "log" + "os" + "path" + "strconv" + "time" + + "github.com/BurntSushi/toml" +) + +func init() { + log.SetFlags(0) + + flag.Usage = usage + flag.Parse() +} + +func usage() { + log.Printf("Usage: %s < json-file\n", path.Base(os.Args[0])) + flag.PrintDefaults() + + os.Exit(1) +} + +func main() { + if flag.NArg() != 0 { + flag.Usage() + } + + var tmp interface{} + if err := json.NewDecoder(os.Stdin).Decode(&tmp); err != nil { + log.Fatalf("Error decoding JSON: %s", err) + } + + tomlData := translate(tmp) + if err := toml.NewEncoder(os.Stdout).Encode(tomlData); err != nil { + log.Fatalf("Error encoding TOML: %s", err) + } +} + +func translate(typedJson interface{}) interface{} { + switch v := typedJson.(type) { + case map[string]interface{}: + if len(v) == 2 && in("type", v) && in("value", v) { + return untag(v) + } + m := make(map[string]interface{}, len(v)) + for k, v2 := range v { + m[k] = translate(v2) + } + return m + case []interface{}: + tabArray := make([]map[string]interface{}, len(v)) + for i := range v { + if m, ok := translate(v[i]).(map[string]interface{}); ok { + tabArray[i] = m + } else { + log.Fatalf("JSON arrays may only contain objects. This " + + "corresponds to only tables being allowed in " + + "TOML table arrays.") + } + } + return tabArray + } + log.Fatalf("Unrecognized JSON format '%T'.", typedJson) + panic("unreachable") +} + +func untag(typed map[string]interface{}) interface{} { + t := typed["type"].(string) + v := typed["value"] + switch t { + case "string": + return v.(string) + case "integer": + v := v.(string) + n, err := strconv.Atoi(v) + if err != nil { + log.Fatalf("Could not parse '%s' as integer: %s", v, err) + } + return n + case "float": + v := v.(string) + f, err := strconv.ParseFloat(v, 64) + if err != nil { + log.Fatalf("Could not parse '%s' as float64: %s", v, err) + } + return f + case "datetime": + v := v.(string) + t, err := time.Parse("2006-01-02T15:04:05Z", v) + if err != nil { + log.Fatalf("Could not parse '%s' as a datetime: %s", v, err) + } + return t + case "bool": + v := v.(string) + switch v { + case "true": + return true + case "false": + return false + } + log.Fatalf("Could not parse '%s' as a boolean.", v) + case "array": + v := v.([]interface{}) + array := make([]interface{}, len(v)) + for i := range v { + if m, ok := v[i].(map[string]interface{}); ok { + array[i] = untag(m) + } else { + log.Fatalf("Arrays may only contain other arrays or "+ + "primitive values, but found a '%T'.", m) + } + } + return array + } + log.Fatalf("Unrecognized tag type '%s'.", t) + panic("unreachable") +} + +func in(key string, m map[string]interface{}) bool { + _, ok := m[key] + return ok +} diff --git a/Godeps/_workspace/src/github.com/BurntSushi/toml/cmd/tomlv/COPYING b/Godeps/_workspace/src/github.com/BurntSushi/toml/cmd/tomlv/COPYING new file mode 100644 index 000000000..5a8e33254 --- /dev/null +++ b/Godeps/_workspace/src/github.com/BurntSushi/toml/cmd/tomlv/COPYING @@ -0,0 +1,14 @@ + DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE + Version 2, December 2004 + + Copyright (C) 2004 Sam Hocevar + + Everyone is permitted to copy and distribute verbatim or modified + copies of this license document, and changing it is allowed as long + as the name is changed. + + DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. You just DO WHAT THE FUCK YOU WANT TO. + diff --git a/Godeps/_workspace/src/github.com/BurntSushi/toml/cmd/tomlv/README.md b/Godeps/_workspace/src/github.com/BurntSushi/toml/cmd/tomlv/README.md new file mode 100644 index 000000000..5df0dc32b --- /dev/null +++ b/Godeps/_workspace/src/github.com/BurntSushi/toml/cmd/tomlv/README.md @@ -0,0 +1,22 @@ +# TOML Validator + +If Go is installed, it's simple to try it out: + +```bash +go get github.com/BurntSushi/toml/cmd/tomlv +tomlv some-toml-file.toml +``` + +You can see the types of every key in a TOML file with: + +```bash +tomlv -types some-toml-file.toml +``` + +At the moment, only one error message is reported at a time. Error messages +include line numbers. No output means that the files given are valid TOML, or +there is a bug in `tomlv`. + +Compatible with TOML version +[v0.1.0](https://github.com/mojombo/toml/blob/master/versions/toml-v0.1.0.md) + diff --git a/Godeps/_workspace/src/github.com/BurntSushi/toml/cmd/tomlv/main.go b/Godeps/_workspace/src/github.com/BurntSushi/toml/cmd/tomlv/main.go new file mode 100644 index 000000000..c7d689a7e --- /dev/null +++ b/Godeps/_workspace/src/github.com/BurntSushi/toml/cmd/tomlv/main.go @@ -0,0 +1,61 @@ +// Command tomlv validates TOML documents and prints each key's type. +package main + +import ( + "flag" + "fmt" + "log" + "os" + "path" + "strings" + "text/tabwriter" + + "github.com/BurntSushi/toml" +) + +var ( + flagTypes = false +) + +func init() { + log.SetFlags(0) + + flag.BoolVar(&flagTypes, "types", flagTypes, + "When set, the types of every defined key will be shown.") + + flag.Usage = usage + flag.Parse() +} + +func usage() { + log.Printf("Usage: %s toml-file [ toml-file ... ]\n", + path.Base(os.Args[0])) + flag.PrintDefaults() + + os.Exit(1) +} + +func main() { + if flag.NArg() < 1 { + flag.Usage() + } + for _, f := range flag.Args() { + var tmp interface{} + md, err := toml.DecodeFile(f, &tmp) + if err != nil { + log.Fatalf("Error in '%s': %s", f, err) + } + if flagTypes { + printTypes(md) + } + } +} + +func printTypes(md toml.MetaData) { + tabw := tabwriter.NewWriter(os.Stdout, 0, 0, 2, ' ', 0) + for _, key := range md.Keys() { + fmt.Fprintf(tabw, "%s%s\t%s\n", + strings.Repeat(" ", len(key)-1), key, md.Type(key...)) + } + tabw.Flush() +} diff --git a/Godeps/_workspace/src/github.com/BurntSushi/toml/decode.go b/Godeps/_workspace/src/github.com/BurntSushi/toml/decode.go new file mode 100644 index 000000000..6c7d398b8 --- /dev/null +++ b/Godeps/_workspace/src/github.com/BurntSushi/toml/decode.go @@ -0,0 +1,492 @@ +package toml + +import ( + "fmt" + "io" + "io/ioutil" + "math" + "reflect" + "strings" + "time" +) + +var e = fmt.Errorf + +// Unmarshaler is the interface implemented by objects that can unmarshal a +// TOML description of themselves. +type Unmarshaler interface { + UnmarshalTOML(interface{}) error +} + +// Unmarshal decodes the contents of `p` in TOML format into a pointer `v`. +func Unmarshal(p []byte, v interface{}) error { + _, err := Decode(string(p), v) + return err +} + +// Primitive is a TOML value that hasn't been decoded into a Go value. +// When using the various `Decode*` functions, the type `Primitive` may +// be given to any value, and its decoding will be delayed. +// +// A `Primitive` value can be decoded using the `PrimitiveDecode` function. +// +// The underlying representation of a `Primitive` value is subject to change. +// Do not rely on it. +// +// N.B. Primitive values are still parsed, so using them will only avoid +// the overhead of reflection. They can be useful when you don't know the +// exact type of TOML data until run time. +type Primitive struct { + undecoded interface{} + context Key +} + +// DEPRECATED! +// +// Use MetaData.PrimitiveDecode instead. +func PrimitiveDecode(primValue Primitive, v interface{}) error { + md := MetaData{decoded: make(map[string]bool)} + return md.unify(primValue.undecoded, rvalue(v)) +} + +// PrimitiveDecode is just like the other `Decode*` functions, except it +// decodes a TOML value that has already been parsed. Valid primitive values +// can *only* be obtained from values filled by the decoder functions, +// including this method. (i.e., `v` may contain more `Primitive` +// values.) +// +// Meta data for primitive values is included in the meta data returned by +// the `Decode*` functions with one exception: keys returned by the Undecoded +// method will only reflect keys that were decoded. Namely, any keys hidden +// behind a Primitive will be considered undecoded. Executing this method will +// update the undecoded keys in the meta data. (See the example.) +func (md *MetaData) PrimitiveDecode(primValue Primitive, v interface{}) error { + md.context = primValue.context + defer func() { md.context = nil }() + return md.unify(primValue.undecoded, rvalue(v)) +} + +// Decode will decode the contents of `data` in TOML format into a pointer +// `v`. +// +// TOML hashes correspond to Go structs or maps. (Dealer's choice. They can be +// used interchangeably.) +// +// TOML arrays of tables correspond to either a slice of structs or a slice +// of maps. +// +// TOML datetimes correspond to Go `time.Time` values. +// +// All other TOML types (float, string, int, bool and array) correspond +// to the obvious Go types. +// +// An exception to the above rules is if a type implements the +// encoding.TextUnmarshaler interface. In this case, any primitive TOML value +// (floats, strings, integers, booleans and datetimes) will be converted to +// a byte string and given to the value's UnmarshalText method. See the +// Unmarshaler example for a demonstration with time duration strings. +// +// Key mapping +// +// TOML keys can map to either keys in a Go map or field names in a Go +// struct. The special `toml` struct tag may be used to map TOML keys to +// struct fields that don't match the key name exactly. (See the example.) +// A case insensitive match to struct names will be tried if an exact match +// can't be found. +// +// The mapping between TOML values and Go values is loose. That is, there +// may exist TOML values that cannot be placed into your representation, and +// there may be parts of your representation that do not correspond to +// TOML values. This loose mapping can be made stricter by using the IsDefined +// and/or Undecoded methods on the MetaData returned. +// +// This decoder will not handle cyclic types. If a cyclic type is passed, +// `Decode` will not terminate. +func Decode(data string, v interface{}) (MetaData, error) { + p, err := parse(data) + if err != nil { + return MetaData{}, err + } + md := MetaData{ + p.mapping, p.types, p.ordered, + make(map[string]bool, len(p.ordered)), nil, + } + return md, md.unify(p.mapping, rvalue(v)) +} + +// DecodeFile is just like Decode, except it will automatically read the +// contents of the file at `fpath` and decode it for you. +func DecodeFile(fpath string, v interface{}) (MetaData, error) { + bs, err := ioutil.ReadFile(fpath) + if err != nil { + return MetaData{}, err + } + return Decode(string(bs), v) +} + +// DecodeReader is just like Decode, except it will consume all bytes +// from the reader and decode it for you. +func DecodeReader(r io.Reader, v interface{}) (MetaData, error) { + bs, err := ioutil.ReadAll(r) + if err != nil { + return MetaData{}, err + } + return Decode(string(bs), v) +} + +// unify performs a sort of type unification based on the structure of `rv`, +// which is the client representation. +// +// Any type mismatch produces an error. Finding a type that we don't know +// how to handle produces an unsupported type error. +func (md *MetaData) unify(data interface{}, rv reflect.Value) error { + + // Special case. Look for a `Primitive` value. + if rv.Type() == reflect.TypeOf((*Primitive)(nil)).Elem() { + // Save the undecoded data and the key context into the primitive + // value. + context := make(Key, len(md.context)) + copy(context, md.context) + rv.Set(reflect.ValueOf(Primitive{ + undecoded: data, + context: context, + })) + return nil + } + + // Special case. Unmarshaler Interface support. + if rv.CanAddr() { + if v, ok := rv.Addr().Interface().(Unmarshaler); ok { + return v.UnmarshalTOML(data) + } + } + + // Special case. Handle time.Time values specifically. + // TODO: Remove this code when we decide to drop support for Go 1.1. + // This isn't necessary in Go 1.2 because time.Time satisfies the encoding + // interfaces. + if rv.Type().AssignableTo(rvalue(time.Time{}).Type()) { + return md.unifyDatetime(data, rv) + } + + // Special case. Look for a value satisfying the TextUnmarshaler interface. + if v, ok := rv.Interface().(TextUnmarshaler); ok { + return md.unifyText(data, v) + } + // BUG(burntsushi) + // The behavior here is incorrect whenever a Go type satisfies the + // encoding.TextUnmarshaler interface but also corresponds to a TOML + // hash or array. In particular, the unmarshaler should only be applied + // to primitive TOML values. But at this point, it will be applied to + // all kinds of values and produce an incorrect error whenever those values + // are hashes or arrays (including arrays of tables). + + k := rv.Kind() + + // laziness + if k >= reflect.Int && k <= reflect.Uint64 { + return md.unifyInt(data, rv) + } + switch k { + case reflect.Ptr: + elem := reflect.New(rv.Type().Elem()) + err := md.unify(data, reflect.Indirect(elem)) + if err != nil { + return err + } + rv.Set(elem) + return nil + case reflect.Struct: + return md.unifyStruct(data, rv) + case reflect.Map: + return md.unifyMap(data, rv) + case reflect.Array: + return md.unifyArray(data, rv) + case reflect.Slice: + return md.unifySlice(data, rv) + case reflect.String: + return md.unifyString(data, rv) + case reflect.Bool: + return md.unifyBool(data, rv) + case reflect.Interface: + // we only support empty interfaces. + if rv.NumMethod() > 0 { + return e("Unsupported type '%s'.", rv.Kind()) + } + return md.unifyAnything(data, rv) + case reflect.Float32: + fallthrough + case reflect.Float64: + return md.unifyFloat64(data, rv) + } + return e("Unsupported type '%s'.", rv.Kind()) +} + +func (md *MetaData) unifyStruct(mapping interface{}, rv reflect.Value) error { + tmap, ok := mapping.(map[string]interface{}) + if !ok { + return mismatch(rv, "map", mapping) + } + + for key, datum := range tmap { + var f *field + fields := cachedTypeFields(rv.Type()) + for i := range fields { + ff := &fields[i] + if ff.name == key { + f = ff + break + } + if f == nil && strings.EqualFold(ff.name, key) { + f = ff + } + } + if f != nil { + subv := rv + for _, i := range f.index { + subv = indirect(subv.Field(i)) + } + if isUnifiable(subv) { + md.decoded[md.context.add(key).String()] = true + md.context = append(md.context, key) + if err := md.unify(datum, subv); err != nil { + return e("Type mismatch for '%s.%s': %s", + rv.Type().String(), f.name, err) + } + md.context = md.context[0 : len(md.context)-1] + } else if f.name != "" { + // Bad user! No soup for you! + return e("Field '%s.%s' is unexported, and therefore cannot "+ + "be loaded with reflection.", rv.Type().String(), f.name) + } + } + } + return nil +} + +func (md *MetaData) unifyMap(mapping interface{}, rv reflect.Value) error { + tmap, ok := mapping.(map[string]interface{}) + if !ok { + return badtype("map", mapping) + } + if rv.IsNil() { + rv.Set(reflect.MakeMap(rv.Type())) + } + for k, v := range tmap { + md.decoded[md.context.add(k).String()] = true + md.context = append(md.context, k) + + rvkey := indirect(reflect.New(rv.Type().Key())) + rvval := reflect.Indirect(reflect.New(rv.Type().Elem())) + if err := md.unify(v, rvval); err != nil { + return err + } + md.context = md.context[0 : len(md.context)-1] + + rvkey.SetString(k) + rv.SetMapIndex(rvkey, rvval) + } + return nil +} + +func (md *MetaData) unifyArray(data interface{}, rv reflect.Value) error { + datav := reflect.ValueOf(data) + if datav.Kind() != reflect.Slice { + return badtype("slice", data) + } + sliceLen := datav.Len() + if sliceLen != rv.Len() { + return e("expected array length %d; got TOML array of length %d", + rv.Len(), sliceLen) + } + return md.unifySliceArray(datav, rv) +} + +func (md *MetaData) unifySlice(data interface{}, rv reflect.Value) error { + datav := reflect.ValueOf(data) + if datav.Kind() != reflect.Slice { + return badtype("slice", data) + } + sliceLen := datav.Len() + if rv.IsNil() { + rv.Set(reflect.MakeSlice(rv.Type(), sliceLen, sliceLen)) + } + return md.unifySliceArray(datav, rv) +} + +func (md *MetaData) unifySliceArray(data, rv reflect.Value) error { + sliceLen := data.Len() + for i := 0; i < sliceLen; i++ { + v := data.Index(i).Interface() + sliceval := indirect(rv.Index(i)) + if err := md.unify(v, sliceval); err != nil { + return err + } + } + return nil +} + +func (md *MetaData) unifyDatetime(data interface{}, rv reflect.Value) error { + if _, ok := data.(time.Time); ok { + rv.Set(reflect.ValueOf(data)) + return nil + } + return badtype("time.Time", data) +} + +func (md *MetaData) unifyString(data interface{}, rv reflect.Value) error { + if s, ok := data.(string); ok { + rv.SetString(s) + return nil + } + return badtype("string", data) +} + +func (md *MetaData) unifyFloat64(data interface{}, rv reflect.Value) error { + if num, ok := data.(float64); ok { + switch rv.Kind() { + case reflect.Float32: + fallthrough + case reflect.Float64: + rv.SetFloat(num) + default: + panic("bug") + } + return nil + } + return badtype("float", data) +} + +func (md *MetaData) unifyInt(data interface{}, rv reflect.Value) error { + if num, ok := data.(int64); ok { + if rv.Kind() >= reflect.Int && rv.Kind() <= reflect.Int64 { + switch rv.Kind() { + case reflect.Int, reflect.Int64: + // No bounds checking necessary. + case reflect.Int8: + if num < math.MinInt8 || num > math.MaxInt8 { + return e("Value '%d' is out of range for int8.", num) + } + case reflect.Int16: + if num < math.MinInt16 || num > math.MaxInt16 { + return e("Value '%d' is out of range for int16.", num) + } + case reflect.Int32: + if num < math.MinInt32 || num > math.MaxInt32 { + return e("Value '%d' is out of range for int32.", num) + } + } + rv.SetInt(num) + } else if rv.Kind() >= reflect.Uint && rv.Kind() <= reflect.Uint64 { + unum := uint64(num) + switch rv.Kind() { + case reflect.Uint, reflect.Uint64: + // No bounds checking necessary. + case reflect.Uint8: + if num < 0 || unum > math.MaxUint8 { + return e("Value '%d' is out of range for uint8.", num) + } + case reflect.Uint16: + if num < 0 || unum > math.MaxUint16 { + return e("Value '%d' is out of range for uint16.", num) + } + case reflect.Uint32: + if num < 0 || unum > math.MaxUint32 { + return e("Value '%d' is out of range for uint32.", num) + } + } + rv.SetUint(unum) + } else { + panic("unreachable") + } + return nil + } + return badtype("integer", data) +} + +func (md *MetaData) unifyBool(data interface{}, rv reflect.Value) error { + if b, ok := data.(bool); ok { + rv.SetBool(b) + return nil + } + return badtype("boolean", data) +} + +func (md *MetaData) unifyAnything(data interface{}, rv reflect.Value) error { + rv.Set(reflect.ValueOf(data)) + return nil +} + +func (md *MetaData) unifyText(data interface{}, v TextUnmarshaler) error { + var s string + switch sdata := data.(type) { + case TextMarshaler: + text, err := sdata.MarshalText() + if err != nil { + return err + } + s = string(text) + case fmt.Stringer: + s = sdata.String() + case string: + s = sdata + case bool: + s = fmt.Sprintf("%v", sdata) + case int64: + s = fmt.Sprintf("%d", sdata) + case float64: + s = fmt.Sprintf("%f", sdata) + default: + return badtype("primitive (string-like)", data) + } + if err := v.UnmarshalText([]byte(s)); err != nil { + return err + } + return nil +} + +// rvalue returns a reflect.Value of `v`. All pointers are resolved. +func rvalue(v interface{}) reflect.Value { + return indirect(reflect.ValueOf(v)) +} + +// indirect returns the value pointed to by a pointer. +// Pointers are followed until the value is not a pointer. +// New values are allocated for each nil pointer. +// +// An exception to this rule is if the value satisfies an interface of +// interest to us (like encoding.TextUnmarshaler). +func indirect(v reflect.Value) reflect.Value { + if v.Kind() != reflect.Ptr { + if v.CanAddr() { + pv := v.Addr() + if _, ok := pv.Interface().(TextUnmarshaler); ok { + return pv + } + } + return v + } + if v.IsNil() { + v.Set(reflect.New(v.Type().Elem())) + } + return indirect(reflect.Indirect(v)) +} + +func isUnifiable(rv reflect.Value) bool { + if rv.CanSet() { + return true + } + if _, ok := rv.Interface().(TextUnmarshaler); ok { + return true + } + return false +} + +func badtype(expected string, data interface{}) error { + return e("Expected %s but found '%T'.", expected, data) +} + +func mismatch(user reflect.Value, expected string, data interface{}) error { + return e("Type mismatch for %s. Expected %s but found '%T'.", + user.Type().String(), expected, data) +} diff --git a/Godeps/_workspace/src/github.com/BurntSushi/toml/decode_meta.go b/Godeps/_workspace/src/github.com/BurntSushi/toml/decode_meta.go new file mode 100644 index 000000000..ef6f545fa --- /dev/null +++ b/Godeps/_workspace/src/github.com/BurntSushi/toml/decode_meta.go @@ -0,0 +1,122 @@ +package toml + +import "strings" + +// MetaData allows access to meta information about TOML data that may not +// be inferrable via reflection. In particular, whether a key has been defined +// and the TOML type of a key. +type MetaData struct { + mapping map[string]interface{} + types map[string]tomlType + keys []Key + decoded map[string]bool + context Key // Used only during decoding. +} + +// IsDefined returns true if the key given exists in the TOML data. The key +// should be specified hierarchially. e.g., +// +// // access the TOML key 'a.b.c' +// IsDefined("a", "b", "c") +// +// IsDefined will return false if an empty key given. Keys are case sensitive. +func (md *MetaData) IsDefined(key ...string) bool { + if len(key) == 0 { + return false + } + + var hash map[string]interface{} + var ok bool + var hashOrVal interface{} = md.mapping + for _, k := range key { + if hash, ok = hashOrVal.(map[string]interface{}); !ok { + return false + } + if hashOrVal, ok = hash[k]; !ok { + return false + } + } + return true +} + +// Type returns a string representation of the type of the key specified. +// +// Type will return the empty string if given an empty key or a key that +// does not exist. Keys are case sensitive. +func (md *MetaData) Type(key ...string) string { + fullkey := strings.Join(key, ".") + if typ, ok := md.types[fullkey]; ok { + return typ.typeString() + } + return "" +} + +// Key is the type of any TOML key, including key groups. Use (MetaData).Keys +// to get values of this type. +type Key []string + +func (k Key) String() string { + return strings.Join(k, ".") +} + +func (k Key) maybeQuotedAll() string { + var ss []string + for i := range k { + ss = append(ss, k.maybeQuoted(i)) + } + return strings.Join(ss, ".") +} + +func (k Key) maybeQuoted(i int) string { + quote := false + for _, c := range k[i] { + if !isBareKeyChar(c) { + quote = true + break + } + } + if quote { + return "\"" + strings.Replace(k[i], "\"", "\\\"", -1) + "\"" + } else { + return k[i] + } +} + +func (k Key) add(piece string) Key { + newKey := make(Key, len(k)+1) + copy(newKey, k) + newKey[len(k)] = piece + return newKey +} + +// Keys returns a slice of every key in the TOML data, including key groups. +// Each key is itself a slice, where the first element is the top of the +// hierarchy and the last is the most specific. +// +// The list will have the same order as the keys appeared in the TOML data. +// +// All keys returned are non-empty. +func (md *MetaData) Keys() []Key { + return md.keys +} + +// Undecoded returns all keys that have not been decoded in the order in which +// they appear in the original TOML document. +// +// This includes keys that haven't been decoded because of a Primitive value. +// Once the Primitive value is decoded, the keys will be considered decoded. +// +// Also note that decoding into an empty interface will result in no decoding, +// and so no keys will be considered decoded. +// +// In this sense, the Undecoded keys correspond to keys in the TOML document +// that do not have a concrete type in your representation. +func (md *MetaData) Undecoded() []Key { + undecoded := make([]Key, 0, len(md.keys)) + for _, key := range md.keys { + if !md.decoded[key.String()] { + undecoded = append(undecoded, key) + } + } + return undecoded +} diff --git a/Godeps/_workspace/src/github.com/BurntSushi/toml/decode_test.go b/Godeps/_workspace/src/github.com/BurntSushi/toml/decode_test.go new file mode 100644 index 000000000..3805931f2 --- /dev/null +++ b/Godeps/_workspace/src/github.com/BurntSushi/toml/decode_test.go @@ -0,0 +1,950 @@ +package toml + +import ( + "fmt" + "log" + "reflect" + "testing" + "time" +) + +func init() { + log.SetFlags(0) +} + +func TestDecodeSimple(t *testing.T) { + var testSimple = ` +age = 250 +andrew = "gallant" +kait = "brady" +now = 1987-07-05T05:45:00Z +yesOrNo = true +pi = 3.14 +colors = [ + ["red", "green", "blue"], + ["cyan", "magenta", "yellow", "black"], +] + +[My.Cats] +plato = "cat 1" +cauchy = "cat 2" +` + + type cats struct { + Plato string + Cauchy string + } + type simple struct { + Age int + Colors [][]string + Pi float64 + YesOrNo bool + Now time.Time + Andrew string + Kait string + My map[string]cats + } + + var val simple + _, err := Decode(testSimple, &val) + if err != nil { + t.Fatal(err) + } + + now, err := time.Parse("2006-01-02T15:04:05", "1987-07-05T05:45:00") + if err != nil { + panic(err) + } + var answer = simple{ + Age: 250, + Andrew: "gallant", + Kait: "brady", + Now: now, + YesOrNo: true, + Pi: 3.14, + Colors: [][]string{ + {"red", "green", "blue"}, + {"cyan", "magenta", "yellow", "black"}, + }, + My: map[string]cats{ + "Cats": cats{Plato: "cat 1", Cauchy: "cat 2"}, + }, + } + if !reflect.DeepEqual(val, answer) { + t.Fatalf("Expected\n-----\n%#v\n-----\nbut got\n-----\n%#v\n", + answer, val) + } +} + +func TestDecodeEmbedded(t *testing.T) { + type Dog struct{ Name string } + type Age int + + tests := map[string]struct { + input string + decodeInto interface{} + wantDecoded interface{} + }{ + "embedded struct": { + input: `Name = "milton"`, + decodeInto: &struct{ Dog }{}, + wantDecoded: &struct{ Dog }{Dog{"milton"}}, + }, + "embedded non-nil pointer to struct": { + input: `Name = "milton"`, + decodeInto: &struct{ *Dog }{}, + wantDecoded: &struct{ *Dog }{&Dog{"milton"}}, + }, + "embedded nil pointer to struct": { + input: ``, + decodeInto: &struct{ *Dog }{}, + wantDecoded: &struct{ *Dog }{nil}, + }, + "embedded int": { + input: `Age = -5`, + decodeInto: &struct{ Age }{}, + wantDecoded: &struct{ Age }{-5}, + }, + } + + for label, test := range tests { + _, err := Decode(test.input, test.decodeInto) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(test.wantDecoded, test.decodeInto) { + t.Errorf("%s: want decoded == %+v, got %+v", + label, test.wantDecoded, test.decodeInto) + } + } +} + +func TestTableArrays(t *testing.T) { + var tomlTableArrays = ` +[[albums]] +name = "Born to Run" + + [[albums.songs]] + name = "Jungleland" + + [[albums.songs]] + name = "Meeting Across the River" + +[[albums]] +name = "Born in the USA" + + [[albums.songs]] + name = "Glory Days" + + [[albums.songs]] + name = "Dancing in the Dark" +` + + type Song struct { + Name string + } + + type Album struct { + Name string + Songs []Song + } + + type Music struct { + Albums []Album + } + + expected := Music{[]Album{ + {"Born to Run", []Song{{"Jungleland"}, {"Meeting Across the River"}}}, + {"Born in the USA", []Song{{"Glory Days"}, {"Dancing in the Dark"}}}, + }} + var got Music + if _, err := Decode(tomlTableArrays, &got); err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(expected, got) { + t.Fatalf("\n%#v\n!=\n%#v\n", expected, got) + } +} + +// Case insensitive matching tests. +// A bit more comprehensive than needed given the current implementation, +// but implementations change. +// Probably still missing demonstrations of some ugly corner cases regarding +// case insensitive matching and multiple fields. +func TestCase(t *testing.T) { + var caseToml = ` +tOpString = "string" +tOpInt = 1 +tOpFloat = 1.1 +tOpBool = true +tOpdate = 2006-01-02T15:04:05Z +tOparray = [ "array" ] +Match = "i should be in Match only" +MatcH = "i should be in MatcH only" +once = "just once" +[nEst.eD] +nEstedString = "another string" +` + + type InsensitiveEd struct { + NestedString string + } + + type InsensitiveNest struct { + Ed InsensitiveEd + } + + type Insensitive struct { + TopString string + TopInt int + TopFloat float64 + TopBool bool + TopDate time.Time + TopArray []string + Match string + MatcH string + Once string + OncE string + Nest InsensitiveNest + } + + tme, err := time.Parse(time.RFC3339, time.RFC3339[:len(time.RFC3339)-5]) + if err != nil { + panic(err) + } + expected := Insensitive{ + TopString: "string", + TopInt: 1, + TopFloat: 1.1, + TopBool: true, + TopDate: tme, + TopArray: []string{"array"}, + MatcH: "i should be in MatcH only", + Match: "i should be in Match only", + Once: "just once", + OncE: "", + Nest: InsensitiveNest{ + Ed: InsensitiveEd{NestedString: "another string"}, + }, + } + var got Insensitive + if _, err := Decode(caseToml, &got); err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(expected, got) { + t.Fatalf("\n%#v\n!=\n%#v\n", expected, got) + } +} + +func TestPointers(t *testing.T) { + type Object struct { + Type string + Description string + } + + type Dict struct { + NamedObject map[string]*Object + BaseObject *Object + Strptr *string + Strptrs []*string + } + s1, s2, s3 := "blah", "abc", "def" + expected := &Dict{ + Strptr: &s1, + Strptrs: []*string{&s2, &s3}, + NamedObject: map[string]*Object{ + "foo": {"FOO", "fooooo!!!"}, + "bar": {"BAR", "ba-ba-ba-ba-barrrr!!!"}, + }, + BaseObject: &Object{"BASE", "da base"}, + } + + ex1 := ` +Strptr = "blah" +Strptrs = ["abc", "def"] + +[NamedObject.foo] +Type = "FOO" +Description = "fooooo!!!" + +[NamedObject.bar] +Type = "BAR" +Description = "ba-ba-ba-ba-barrrr!!!" + +[BaseObject] +Type = "BASE" +Description = "da base" +` + dict := new(Dict) + _, err := Decode(ex1, dict) + if err != nil { + t.Errorf("Decode error: %v", err) + } + if !reflect.DeepEqual(expected, dict) { + t.Fatalf("\n%#v\n!=\n%#v\n", expected, dict) + } +} + +type sphere struct { + Center [3]float64 + Radius float64 +} + +func TestDecodeSimpleArray(t *testing.T) { + var s1 sphere + if _, err := Decode(`center = [0.0, 1.5, 0.0]`, &s1); err != nil { + t.Fatal(err) + } +} + +func TestDecodeArrayWrongSize(t *testing.T) { + var s1 sphere + if _, err := Decode(`center = [0.1, 2.3]`, &s1); err == nil { + t.Fatal("Expected array type mismatch error") + } +} + +func TestDecodeLargeIntoSmallInt(t *testing.T) { + type table struct { + Value int8 + } + var tab table + if _, err := Decode(`value = 500`, &tab); err == nil { + t.Fatal("Expected integer out-of-bounds error.") + } +} + +func TestDecodeSizedInts(t *testing.T) { + type table struct { + U8 uint8 + U16 uint16 + U32 uint32 + U64 uint64 + U uint + I8 int8 + I16 int16 + I32 int32 + I64 int64 + I int + } + answer := table{1, 1, 1, 1, 1, -1, -1, -1, -1, -1} + toml := ` + u8 = 1 + u16 = 1 + u32 = 1 + u64 = 1 + u = 1 + i8 = -1 + i16 = -1 + i32 = -1 + i64 = -1 + i = -1 + ` + var tab table + if _, err := Decode(toml, &tab); err != nil { + t.Fatal(err.Error()) + } + if answer != tab { + t.Fatalf("Expected %#v but got %#v", answer, tab) + } +} + +func TestUnmarshaler(t *testing.T) { + + var tomlBlob = ` +[dishes.hamboogie] +name = "Hamboogie with fries" +price = 10.99 + +[[dishes.hamboogie.ingredients]] +name = "Bread Bun" + +[[dishes.hamboogie.ingredients]] +name = "Lettuce" + +[[dishes.hamboogie.ingredients]] +name = "Real Beef Patty" + +[[dishes.hamboogie.ingredients]] +name = "Tomato" + +[dishes.eggsalad] +name = "Egg Salad with rice" +price = 3.99 + +[[dishes.eggsalad.ingredients]] +name = "Egg" + +[[dishes.eggsalad.ingredients]] +name = "Mayo" + +[[dishes.eggsalad.ingredients]] +name = "Rice" +` + m := &menu{} + if _, err := Decode(tomlBlob, m); err != nil { + log.Fatal(err) + } + + if len(m.Dishes) != 2 { + t.Log("two dishes should be loaded with UnmarshalTOML()") + t.Errorf("expected %d but got %d", 2, len(m.Dishes)) + } + + eggSalad := m.Dishes["eggsalad"] + if _, ok := interface{}(eggSalad).(dish); !ok { + t.Errorf("expected a dish") + } + + if eggSalad.Name != "Egg Salad with rice" { + t.Errorf("expected the dish to be named 'Egg Salad with rice'") + } + + if len(eggSalad.Ingredients) != 3 { + t.Log("dish should be loaded with UnmarshalTOML()") + t.Errorf("expected %d but got %d", 3, len(eggSalad.Ingredients)) + } + + found := false + for _, i := range eggSalad.Ingredients { + if i.Name == "Rice" { + found = true + break + } + } + if !found { + t.Error("Rice was not loaded in UnmarshalTOML()") + } + + // test on a value - must be passed as * + o := menu{} + if _, err := Decode(tomlBlob, &o); err != nil { + log.Fatal(err) + } + +} + +type menu struct { + Dishes map[string]dish +} + +func (m *menu) UnmarshalTOML(p interface{}) error { + m.Dishes = make(map[string]dish) + data, _ := p.(map[string]interface{}) + dishes := data["dishes"].(map[string]interface{}) + for n, v := range dishes { + if d, ok := v.(map[string]interface{}); ok { + nd := dish{} + nd.UnmarshalTOML(d) + m.Dishes[n] = nd + } else { + return fmt.Errorf("not a dish") + } + } + return nil +} + +type dish struct { + Name string + Price float32 + Ingredients []ingredient +} + +func (d *dish) UnmarshalTOML(p interface{}) error { + data, _ := p.(map[string]interface{}) + d.Name, _ = data["name"].(string) + d.Price, _ = data["price"].(float32) + ingredients, _ := data["ingredients"].([]map[string]interface{}) + for _, e := range ingredients { + n, _ := interface{}(e).(map[string]interface{}) + name, _ := n["name"].(string) + i := ingredient{name} + d.Ingredients = append(d.Ingredients, i) + } + return nil +} + +type ingredient struct { + Name string +} + +func ExampleMetaData_PrimitiveDecode() { + var md MetaData + var err error + + var tomlBlob = ` +ranking = ["Springsteen", "J Geils"] + +[bands.Springsteen] +started = 1973 +albums = ["Greetings", "WIESS", "Born to Run", "Darkness"] + +[bands."J Geils"] +started = 1970 +albums = ["The J. Geils Band", "Full House", "Blow Your Face Out"] +` + + type band struct { + Started int + Albums []string + } + type classics struct { + Ranking []string + Bands map[string]Primitive + } + + // Do the initial decode. Reflection is delayed on Primitive values. + var music classics + if md, err = Decode(tomlBlob, &music); err != nil { + log.Fatal(err) + } + + // MetaData still includes information on Primitive values. + fmt.Printf("Is `bands.Springsteen` defined? %v\n", + md.IsDefined("bands", "Springsteen")) + + // Decode primitive data into Go values. + for _, artist := range music.Ranking { + // A band is a primitive value, so we need to decode it to get a + // real `band` value. + primValue := music.Bands[artist] + + var aBand band + if err = md.PrimitiveDecode(primValue, &aBand); err != nil { + log.Fatal(err) + } + fmt.Printf("%s started in %d.\n", artist, aBand.Started) + } + // Check to see if there were any fields left undecoded. + // Note that this won't be empty before decoding the Primitive value! + fmt.Printf("Undecoded: %q\n", md.Undecoded()) + + // Output: + // Is `bands.Springsteen` defined? true + // Springsteen started in 1973. + // J Geils started in 1970. + // Undecoded: [] +} + +func ExampleDecode() { + var tomlBlob = ` +# Some comments. +[alpha] +ip = "10.0.0.1" + + [alpha.config] + Ports = [ 8001, 8002 ] + Location = "Toronto" + Created = 1987-07-05T05:45:00Z + +[beta] +ip = "10.0.0.2" + + [beta.config] + Ports = [ 9001, 9002 ] + Location = "New Jersey" + Created = 1887-01-05T05:55:00Z +` + + type serverConfig struct { + Ports []int + Location string + Created time.Time + } + + type server struct { + IP string `toml:"ip"` + Config serverConfig `toml:"config"` + } + + type servers map[string]server + + var config servers + if _, err := Decode(tomlBlob, &config); err != nil { + log.Fatal(err) + } + + for _, name := range []string{"alpha", "beta"} { + s := config[name] + fmt.Printf("Server: %s (ip: %s) in %s created on %s\n", + name, s.IP, s.Config.Location, + s.Config.Created.Format("2006-01-02")) + fmt.Printf("Ports: %v\n", s.Config.Ports) + } + + // Output: + // Server: alpha (ip: 10.0.0.1) in Toronto created on 1987-07-05 + // Ports: [8001 8002] + // Server: beta (ip: 10.0.0.2) in New Jersey created on 1887-01-05 + // Ports: [9001 9002] +} + +type duration struct { + time.Duration +} + +func (d *duration) UnmarshalText(text []byte) error { + var err error + d.Duration, err = time.ParseDuration(string(text)) + return err +} + +// Example Unmarshaler shows how to decode TOML strings into your own +// custom data type. +func Example_unmarshaler() { + blob := ` +[[song]] +name = "Thunder Road" +duration = "4m49s" + +[[song]] +name = "Stairway to Heaven" +duration = "8m03s" +` + type song struct { + Name string + Duration duration + } + type songs struct { + Song []song + } + var favorites songs + if _, err := Decode(blob, &favorites); err != nil { + log.Fatal(err) + } + + // Code to implement the TextUnmarshaler interface for `duration`: + // + // type duration struct { + // time.Duration + // } + // + // func (d *duration) UnmarshalText(text []byte) error { + // var err error + // d.Duration, err = time.ParseDuration(string(text)) + // return err + // } + + for _, s := range favorites.Song { + fmt.Printf("%s (%s)\n", s.Name, s.Duration) + } + // Output: + // Thunder Road (4m49s) + // Stairway to Heaven (8m3s) +} + +// Example StrictDecoding shows how to detect whether there are keys in the +// TOML document that weren't decoded into the value given. This is useful +// for returning an error to the user if they've included extraneous fields +// in their configuration. +func Example_strictDecoding() { + var blob = ` +key1 = "value1" +key2 = "value2" +key3 = "value3" +` + type config struct { + Key1 string + Key3 string + } + + var conf config + md, err := Decode(blob, &conf) + if err != nil { + log.Fatal(err) + } + fmt.Printf("Undecoded keys: %q\n", md.Undecoded()) + // Output: + // Undecoded keys: ["key2"] +} + +// Example UnmarshalTOML shows how to implement a struct type that knows how to +// unmarshal itself. The struct must take full responsibility for mapping the +// values passed into the struct. The method may be used with interfaces in a +// struct in cases where the actual type is not known until the data is +// examined. +func Example_unmarshalTOML() { + + var blob = ` +[[parts]] +type = "valve" +id = "valve-1" +size = 1.2 +rating = 4 + +[[parts]] +type = "valve" +id = "valve-2" +size = 2.1 +rating = 5 + +[[parts]] +type = "pipe" +id = "pipe-1" +length = 2.1 +diameter = 12 + +[[parts]] +type = "cable" +id = "cable-1" +length = 12 +rating = 3.1 +` + o := &order{} + err := Unmarshal([]byte(blob), o) + if err != nil { + log.Fatal(err) + } + + fmt.Println(len(o.parts)) + + for _, part := range o.parts { + fmt.Println(part.Name()) + } + + // Code to implement UmarshalJSON. + + // type order struct { + // // NOTE `order.parts` is a private slice of type `part` which is an + // // interface and may only be loaded from toml using the + // // UnmarshalTOML() method of the Umarshaler interface. + // parts parts + // } + + // func (o *order) UnmarshalTOML(data interface{}) error { + + // // NOTE the example below contains detailed type casting to show how + // // the 'data' is retrieved. In operational use, a type cast wrapper + // // may be prefered e.g. + // // + // // func AsMap(v interface{}) (map[string]interface{}, error) { + // // return v.(map[string]interface{}) + // // } + // // + // // resulting in: + // // d, _ := AsMap(data) + // // + + // d, _ := data.(map[string]interface{}) + // parts, _ := d["parts"].([]map[string]interface{}) + + // for _, p := range parts { + + // typ, _ := p["type"].(string) + // id, _ := p["id"].(string) + + // // detect the type of part and handle each case + // switch p["type"] { + // case "valve": + + // size := float32(p["size"].(float64)) + // rating := int(p["rating"].(int64)) + + // valve := &valve{ + // Type: typ, + // ID: id, + // Size: size, + // Rating: rating, + // } + + // o.parts = append(o.parts, valve) + + // case "pipe": + + // length := float32(p["length"].(float64)) + // diameter := int(p["diameter"].(int64)) + + // pipe := &pipe{ + // Type: typ, + // ID: id, + // Length: length, + // Diameter: diameter, + // } + + // o.parts = append(o.parts, pipe) + + // case "cable": + + // length := int(p["length"].(int64)) + // rating := float32(p["rating"].(float64)) + + // cable := &cable{ + // Type: typ, + // ID: id, + // Length: length, + // Rating: rating, + // } + + // o.parts = append(o.parts, cable) + + // } + // } + + // return nil + // } + + // type parts []part + + // type part interface { + // Name() string + // } + + // type valve struct { + // Type string + // ID string + // Size float32 + // Rating int + // } + + // func (v *valve) Name() string { + // return fmt.Sprintf("VALVE: %s", v.ID) + // } + + // type pipe struct { + // Type string + // ID string + // Length float32 + // Diameter int + // } + + // func (p *pipe) Name() string { + // return fmt.Sprintf("PIPE: %s", p.ID) + // } + + // type cable struct { + // Type string + // ID string + // Length int + // Rating float32 + // } + + // func (c *cable) Name() string { + // return fmt.Sprintf("CABLE: %s", c.ID) + // } + + // Output: + // 4 + // VALVE: valve-1 + // VALVE: valve-2 + // PIPE: pipe-1 + // CABLE: cable-1 + +} + +type order struct { + // NOTE `order.parts` is a private slice of type `part` which is an + // interface and may only be loaded from toml using the UnmarshalTOML() + // method of the Umarshaler interface. + parts parts +} + +func (o *order) UnmarshalTOML(data interface{}) error { + + // NOTE the example below contains detailed type casting to show how + // the 'data' is retrieved. In operational use, a type cast wrapper + // may be prefered e.g. + // + // func AsMap(v interface{}) (map[string]interface{}, error) { + // return v.(map[string]interface{}) + // } + // + // resulting in: + // d, _ := AsMap(data) + // + + d, _ := data.(map[string]interface{}) + parts, _ := d["parts"].([]map[string]interface{}) + + for _, p := range parts { + + typ, _ := p["type"].(string) + id, _ := p["id"].(string) + + // detect the type of part and handle each case + switch p["type"] { + case "valve": + + size := float32(p["size"].(float64)) + rating := int(p["rating"].(int64)) + + valve := &valve{ + Type: typ, + ID: id, + Size: size, + Rating: rating, + } + + o.parts = append(o.parts, valve) + + case "pipe": + + length := float32(p["length"].(float64)) + diameter := int(p["diameter"].(int64)) + + pipe := &pipe{ + Type: typ, + ID: id, + Length: length, + Diameter: diameter, + } + + o.parts = append(o.parts, pipe) + + case "cable": + + length := int(p["length"].(int64)) + rating := float32(p["rating"].(float64)) + + cable := &cable{ + Type: typ, + ID: id, + Length: length, + Rating: rating, + } + + o.parts = append(o.parts, cable) + + } + } + + return nil +} + +type parts []part + +type part interface { + Name() string +} + +type valve struct { + Type string + ID string + Size float32 + Rating int +} + +func (v *valve) Name() string { + return fmt.Sprintf("VALVE: %s", v.ID) +} + +type pipe struct { + Type string + ID string + Length float32 + Diameter int +} + +func (p *pipe) Name() string { + return fmt.Sprintf("PIPE: %s", p.ID) +} + +type cable struct { + Type string + ID string + Length int + Rating float32 +} + +func (c *cable) Name() string { + return fmt.Sprintf("CABLE: %s", c.ID) +} diff --git a/Godeps/_workspace/src/github.com/BurntSushi/toml/doc.go b/Godeps/_workspace/src/github.com/BurntSushi/toml/doc.go new file mode 100644 index 000000000..fe2680004 --- /dev/null +++ b/Godeps/_workspace/src/github.com/BurntSushi/toml/doc.go @@ -0,0 +1,27 @@ +/* +Package toml provides facilities for decoding and encoding TOML configuration +files via reflection. There is also support for delaying decoding with +the Primitive type, and querying the set of keys in a TOML document with the +MetaData type. + +The specification implemented: https://github.com/mojombo/toml + +The sub-command github.com/BurntSushi/toml/cmd/tomlv can be used to verify +whether a file is a valid TOML document. It can also be used to print the +type of each key in a TOML document. + +Testing + +There are two important types of tests used for this package. The first is +contained inside '*_test.go' files and uses the standard Go unit testing +framework. These tests are primarily devoted to holistically testing the +decoder and encoder. + +The second type of testing is used to verify the implementation's adherence +to the TOML specification. These tests have been factored into their own +project: https://github.com/BurntSushi/toml-test + +The reason the tests are in a separate project is so that they can be used by +any implementation of TOML. Namely, it is language agnostic. +*/ +package toml diff --git a/Godeps/_workspace/src/github.com/BurntSushi/toml/encode.go b/Godeps/_workspace/src/github.com/BurntSushi/toml/encode.go new file mode 100644 index 000000000..64e8c47e1 --- /dev/null +++ b/Godeps/_workspace/src/github.com/BurntSushi/toml/encode.go @@ -0,0 +1,496 @@ +package toml + +import ( + "bufio" + "errors" + "fmt" + "io" + "reflect" + "sort" + "strconv" + "strings" + "time" +) + +type tomlEncodeError struct{ error } + +var ( + errArrayMixedElementTypes = errors.New( + "can't encode array with mixed element types") + errArrayNilElement = errors.New( + "can't encode array with nil element") + errNonString = errors.New( + "can't encode a map with non-string key type") + errAnonNonStruct = errors.New( + "can't encode an anonymous field that is not a struct") + errArrayNoTable = errors.New( + "TOML array element can't contain a table") + errNoKey = errors.New( + "top-level values must be a Go map or struct") + errAnything = errors.New("") // used in testing +) + +var quotedReplacer = strings.NewReplacer( + "\t", "\\t", + "\n", "\\n", + "\r", "\\r", + "\"", "\\\"", + "\\", "\\\\", +) + +// Encoder controls the encoding of Go values to a TOML document to some +// io.Writer. +// +// The indentation level can be controlled with the Indent field. +type Encoder struct { + // A single indentation level. By default it is two spaces. + Indent string + + // hasWritten is whether we have written any output to w yet. + hasWritten bool + w *bufio.Writer +} + +// NewEncoder returns a TOML encoder that encodes Go values to the io.Writer +// given. By default, a single indentation level is 2 spaces. +func NewEncoder(w io.Writer) *Encoder { + return &Encoder{ + w: bufio.NewWriter(w), + Indent: " ", + } +} + +// Encode writes a TOML representation of the Go value to the underlying +// io.Writer. If the value given cannot be encoded to a valid TOML document, +// then an error is returned. +// +// The mapping between Go values and TOML values should be precisely the same +// as for the Decode* functions. Similarly, the TextMarshaler interface is +// supported by encoding the resulting bytes as strings. (If you want to write +// arbitrary binary data then you will need to use something like base64 since +// TOML does not have any binary types.) +// +// When encoding TOML hashes (i.e., Go maps or structs), keys without any +// sub-hashes are encoded first. +// +// If a Go map is encoded, then its keys are sorted alphabetically for +// deterministic output. More control over this behavior may be provided if +// there is demand for it. +// +// Encoding Go values without a corresponding TOML representation---like map +// types with non-string keys---will cause an error to be returned. Similarly +// for mixed arrays/slices, arrays/slices with nil elements, embedded +// non-struct types and nested slices containing maps or structs. +// (e.g., [][]map[string]string is not allowed but []map[string]string is OK +// and so is []map[string][]string.) +func (enc *Encoder) Encode(v interface{}) error { + rv := eindirect(reflect.ValueOf(v)) + if err := enc.safeEncode(Key([]string{}), rv); err != nil { + return err + } + return enc.w.Flush() +} + +func (enc *Encoder) safeEncode(key Key, rv reflect.Value) (err error) { + defer func() { + if r := recover(); r != nil { + if terr, ok := r.(tomlEncodeError); ok { + err = terr.error + return + } + panic(r) + } + }() + enc.encode(key, rv) + return nil +} + +func (enc *Encoder) encode(key Key, rv reflect.Value) { + // Special case. Time needs to be in ISO8601 format. + // Special case. If we can marshal the type to text, then we used that. + // Basically, this prevents the encoder for handling these types as + // generic structs (or whatever the underlying type of a TextMarshaler is). + switch rv.Interface().(type) { + case time.Time, TextMarshaler: + enc.keyEqElement(key, rv) + return + } + + k := rv.Kind() + switch k { + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, + reflect.Int64, + reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, + reflect.Uint64, + reflect.Float32, reflect.Float64, reflect.String, reflect.Bool: + enc.keyEqElement(key, rv) + case reflect.Array, reflect.Slice: + if typeEqual(tomlArrayHash, tomlTypeOfGo(rv)) { + enc.eArrayOfTables(key, rv) + } else { + enc.keyEqElement(key, rv) + } + case reflect.Interface: + if rv.IsNil() { + return + } + enc.encode(key, rv.Elem()) + case reflect.Map: + if rv.IsNil() { + return + } + enc.eTable(key, rv) + case reflect.Ptr: + if rv.IsNil() { + return + } + enc.encode(key, rv.Elem()) + case reflect.Struct: + enc.eTable(key, rv) + default: + panic(e("Unsupported type for key '%s': %s", key, k)) + } +} + +// eElement encodes any value that can be an array element (primitives and +// arrays). +func (enc *Encoder) eElement(rv reflect.Value) { + switch v := rv.Interface().(type) { + case time.Time: + // Special case time.Time as a primitive. Has to come before + // TextMarshaler below because time.Time implements + // encoding.TextMarshaler, but we need to always use UTC. + enc.wf(v.In(time.FixedZone("UTC", 0)).Format("2006-01-02T15:04:05Z")) + return + case TextMarshaler: + // Special case. Use text marshaler if it's available for this value. + if s, err := v.MarshalText(); err != nil { + encPanic(err) + } else { + enc.writeQuoted(string(s)) + } + return + } + switch rv.Kind() { + case reflect.Bool: + enc.wf(strconv.FormatBool(rv.Bool())) + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, + reflect.Int64: + enc.wf(strconv.FormatInt(rv.Int(), 10)) + case reflect.Uint, reflect.Uint8, reflect.Uint16, + reflect.Uint32, reflect.Uint64: + enc.wf(strconv.FormatUint(rv.Uint(), 10)) + case reflect.Float32: + enc.wf(floatAddDecimal(strconv.FormatFloat(rv.Float(), 'f', -1, 32))) + case reflect.Float64: + enc.wf(floatAddDecimal(strconv.FormatFloat(rv.Float(), 'f', -1, 64))) + case reflect.Array, reflect.Slice: + enc.eArrayOrSliceElement(rv) + case reflect.Interface: + enc.eElement(rv.Elem()) + case reflect.String: + enc.writeQuoted(rv.String()) + default: + panic(e("Unexpected primitive type: %s", rv.Kind())) + } +} + +// By the TOML spec, all floats must have a decimal with at least one +// number on either side. +func floatAddDecimal(fstr string) string { + if !strings.Contains(fstr, ".") { + return fstr + ".0" + } + return fstr +} + +func (enc *Encoder) writeQuoted(s string) { + enc.wf("\"%s\"", quotedReplacer.Replace(s)) +} + +func (enc *Encoder) eArrayOrSliceElement(rv reflect.Value) { + length := rv.Len() + enc.wf("[") + for i := 0; i < length; i++ { + elem := rv.Index(i) + enc.eElement(elem) + if i != length-1 { + enc.wf(", ") + } + } + enc.wf("]") +} + +func (enc *Encoder) eArrayOfTables(key Key, rv reflect.Value) { + if len(key) == 0 { + encPanic(errNoKey) + } + for i := 0; i < rv.Len(); i++ { + trv := rv.Index(i) + if isNil(trv) { + continue + } + panicIfInvalidKey(key) + enc.newline() + enc.wf("%s[[%s]]", enc.indentStr(key), key.maybeQuotedAll()) + enc.newline() + enc.eMapOrStruct(key, trv) + } +} + +func (enc *Encoder) eTable(key Key, rv reflect.Value) { + panicIfInvalidKey(key) + if len(key) == 1 { + // Output an extra new line between top-level tables. + // (The newline isn't written if nothing else has been written though.) + enc.newline() + } + if len(key) > 0 { + enc.wf("%s[%s]", enc.indentStr(key), key.maybeQuotedAll()) + enc.newline() + } + enc.eMapOrStruct(key, rv) +} + +func (enc *Encoder) eMapOrStruct(key Key, rv reflect.Value) { + switch rv := eindirect(rv); rv.Kind() { + case reflect.Map: + enc.eMap(key, rv) + case reflect.Struct: + enc.eStruct(key, rv) + default: + panic("eTable: unhandled reflect.Value Kind: " + rv.Kind().String()) + } +} + +func (enc *Encoder) eMap(key Key, rv reflect.Value) { + rt := rv.Type() + if rt.Key().Kind() != reflect.String { + encPanic(errNonString) + } + + // Sort keys so that we have deterministic output. And write keys directly + // underneath this key first, before writing sub-structs or sub-maps. + var mapKeysDirect, mapKeysSub []string + for _, mapKey := range rv.MapKeys() { + k := mapKey.String() + if typeIsHash(tomlTypeOfGo(rv.MapIndex(mapKey))) { + mapKeysSub = append(mapKeysSub, k) + } else { + mapKeysDirect = append(mapKeysDirect, k) + } + } + + var writeMapKeys = func(mapKeys []string) { + sort.Strings(mapKeys) + for _, mapKey := range mapKeys { + mrv := rv.MapIndex(reflect.ValueOf(mapKey)) + if isNil(mrv) { + // Don't write anything for nil fields. + continue + } + enc.encode(key.add(mapKey), mrv) + } + } + writeMapKeys(mapKeysDirect) + writeMapKeys(mapKeysSub) +} + +func (enc *Encoder) eStruct(key Key, rv reflect.Value) { + // Write keys for fields directly under this key first, because if we write + // a field that creates a new table, then all keys under it will be in that + // table (not the one we're writing here). + rt := rv.Type() + var fieldsDirect, fieldsSub [][]int + var addFields func(rt reflect.Type, rv reflect.Value, start []int) + addFields = func(rt reflect.Type, rv reflect.Value, start []int) { + for i := 0; i < rt.NumField(); i++ { + f := rt.Field(i) + // skip unexporded fields + if f.PkgPath != "" { + continue + } + frv := rv.Field(i) + if f.Anonymous { + frv := eindirect(frv) + t := frv.Type() + if t.Kind() != reflect.Struct { + encPanic(errAnonNonStruct) + } + addFields(t, frv, f.Index) + } else if typeIsHash(tomlTypeOfGo(frv)) { + fieldsSub = append(fieldsSub, append(start, f.Index...)) + } else { + fieldsDirect = append(fieldsDirect, append(start, f.Index...)) + } + } + } + addFields(rt, rv, nil) + + var writeFields = func(fields [][]int) { + for _, fieldIndex := range fields { + sft := rt.FieldByIndex(fieldIndex) + sf := rv.FieldByIndex(fieldIndex) + if isNil(sf) { + // Don't write anything for nil fields. + continue + } + + keyName := sft.Tag.Get("toml") + if keyName == "-" { + continue + } + if keyName == "" { + keyName = sft.Name + } + enc.encode(key.add(keyName), sf) + } + } + writeFields(fieldsDirect) + writeFields(fieldsSub) +} + +// tomlTypeName returns the TOML type name of the Go value's type. It is +// used to determine whether the types of array elements are mixed (which is +// forbidden). If the Go value is nil, then it is illegal for it to be an array +// element, and valueIsNil is returned as true. + +// Returns the TOML type of a Go value. The type may be `nil`, which means +// no concrete TOML type could be found. +func tomlTypeOfGo(rv reflect.Value) tomlType { + if isNil(rv) || !rv.IsValid() { + return nil + } + switch rv.Kind() { + case reflect.Bool: + return tomlBool + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, + reflect.Int64, + reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, + reflect.Uint64: + return tomlInteger + case reflect.Float32, reflect.Float64: + return tomlFloat + case reflect.Array, reflect.Slice: + if typeEqual(tomlHash, tomlArrayType(rv)) { + return tomlArrayHash + } else { + return tomlArray + } + case reflect.Ptr, reflect.Interface: + return tomlTypeOfGo(rv.Elem()) + case reflect.String: + return tomlString + case reflect.Map: + return tomlHash + case reflect.Struct: + switch rv.Interface().(type) { + case time.Time: + return tomlDatetime + case TextMarshaler: + return tomlString + default: + return tomlHash + } + default: + panic("unexpected reflect.Kind: " + rv.Kind().String()) + } +} + +// tomlArrayType returns the element type of a TOML array. The type returned +// may be nil if it cannot be determined (e.g., a nil slice or a zero length +// slize). This function may also panic if it finds a type that cannot be +// expressed in TOML (such as nil elements, heterogeneous arrays or directly +// nested arrays of tables). +func tomlArrayType(rv reflect.Value) tomlType { + if isNil(rv) || !rv.IsValid() || rv.Len() == 0 { + return nil + } + firstType := tomlTypeOfGo(rv.Index(0)) + if firstType == nil { + encPanic(errArrayNilElement) + } + + rvlen := rv.Len() + for i := 1; i < rvlen; i++ { + elem := rv.Index(i) + switch elemType := tomlTypeOfGo(elem); { + case elemType == nil: + encPanic(errArrayNilElement) + case !typeEqual(firstType, elemType): + encPanic(errArrayMixedElementTypes) + } + } + // If we have a nested array, then we must make sure that the nested + // array contains ONLY primitives. + // This checks arbitrarily nested arrays. + if typeEqual(firstType, tomlArray) || typeEqual(firstType, tomlArrayHash) { + nest := tomlArrayType(eindirect(rv.Index(0))) + if typeEqual(nest, tomlHash) || typeEqual(nest, tomlArrayHash) { + encPanic(errArrayNoTable) + } + } + return firstType +} + +func (enc *Encoder) newline() { + if enc.hasWritten { + enc.wf("\n") + } +} + +func (enc *Encoder) keyEqElement(key Key, val reflect.Value) { + if len(key) == 0 { + encPanic(errNoKey) + } + panicIfInvalidKey(key) + enc.wf("%s%s = ", enc.indentStr(key), key.maybeQuoted(len(key)-1)) + enc.eElement(val) + enc.newline() +} + +func (enc *Encoder) wf(format string, v ...interface{}) { + if _, err := fmt.Fprintf(enc.w, format, v...); err != nil { + encPanic(err) + } + enc.hasWritten = true +} + +func (enc *Encoder) indentStr(key Key) string { + return strings.Repeat(enc.Indent, len(key)-1) +} + +func encPanic(err error) { + panic(tomlEncodeError{err}) +} + +func eindirect(v reflect.Value) reflect.Value { + switch v.Kind() { + case reflect.Ptr, reflect.Interface: + return eindirect(v.Elem()) + default: + return v + } +} + +func isNil(rv reflect.Value) bool { + switch rv.Kind() { + case reflect.Interface, reflect.Map, reflect.Ptr, reflect.Slice: + return rv.IsNil() + default: + return false + } +} + +func panicIfInvalidKey(key Key) { + for _, k := range key { + if len(k) == 0 { + encPanic(e("Key '%s' is not a valid table name. Key names "+ + "cannot be empty.", key.maybeQuotedAll())) + } + } +} + +func isValidKeyName(s string) bool { + return len(s) != 0 +} diff --git a/Godeps/_workspace/src/github.com/BurntSushi/toml/encode_test.go b/Godeps/_workspace/src/github.com/BurntSushi/toml/encode_test.go new file mode 100644 index 000000000..74a5ee5d2 --- /dev/null +++ b/Godeps/_workspace/src/github.com/BurntSushi/toml/encode_test.go @@ -0,0 +1,506 @@ +package toml + +import ( + "bytes" + "fmt" + "log" + "net" + "testing" + "time" +) + +func TestEncodeRoundTrip(t *testing.T) { + type Config struct { + Age int + Cats []string + Pi float64 + Perfection []int + DOB time.Time + Ipaddress net.IP + } + + var inputs = Config{ + 13, + []string{"one", "two", "three"}, + 3.145, + []int{11, 2, 3, 4}, + time.Now(), + net.ParseIP("192.168.59.254"), + } + + var firstBuffer bytes.Buffer + e := NewEncoder(&firstBuffer) + err := e.Encode(inputs) + if err != nil { + t.Fatal(err) + } + var outputs Config + if _, err := Decode(firstBuffer.String(), &outputs); err != nil { + log.Printf("Could not decode:\n-----\n%s\n-----\n", + firstBuffer.String()) + t.Fatal(err) + } + + // could test each value individually, but I'm lazy + var secondBuffer bytes.Buffer + e2 := NewEncoder(&secondBuffer) + err = e2.Encode(outputs) + if err != nil { + t.Fatal(err) + } + if firstBuffer.String() != secondBuffer.String() { + t.Error( + firstBuffer.String(), + "\n\n is not identical to\n\n", + secondBuffer.String()) + } +} + +// XXX(burntsushi) +// I think these tests probably should be removed. They are good, but they +// ought to be obsolete by toml-test. +func TestEncode(t *testing.T) { + type Embedded struct { + Int int `toml:"_int"` + } + type NonStruct int + + date := time.Date(2014, 5, 11, 20, 30, 40, 0, time.FixedZone("IST", 3600)) + dateStr := "2014-05-11T19:30:40Z" + + tests := map[string]struct { + input interface{} + wantOutput string + wantError error + }{ + "bool field": { + input: struct { + BoolTrue bool + BoolFalse bool + }{true, false}, + wantOutput: "BoolTrue = true\nBoolFalse = false\n", + }, + "int fields": { + input: struct { + Int int + Int8 int8 + Int16 int16 + Int32 int32 + Int64 int64 + }{1, 2, 3, 4, 5}, + wantOutput: "Int = 1\nInt8 = 2\nInt16 = 3\nInt32 = 4\nInt64 = 5\n", + }, + "uint fields": { + input: struct { + Uint uint + Uint8 uint8 + Uint16 uint16 + Uint32 uint32 + Uint64 uint64 + }{1, 2, 3, 4, 5}, + wantOutput: "Uint = 1\nUint8 = 2\nUint16 = 3\nUint32 = 4" + + "\nUint64 = 5\n", + }, + "float fields": { + input: struct { + Float32 float32 + Float64 float64 + }{1.5, 2.5}, + wantOutput: "Float32 = 1.5\nFloat64 = 2.5\n", + }, + "string field": { + input: struct{ String string }{"foo"}, + wantOutput: "String = \"foo\"\n", + }, + "string field and unexported field": { + input: struct { + String string + unexported int + }{"foo", 0}, + wantOutput: "String = \"foo\"\n", + }, + "datetime field in UTC": { + input: struct{ Date time.Time }{date}, + wantOutput: fmt.Sprintf("Date = %s\n", dateStr), + }, + "datetime field as primitive": { + // Using a map here to fail if isStructOrMap() returns true for + // time.Time. + input: map[string]interface{}{ + "Date": date, + "Int": 1, + }, + wantOutput: fmt.Sprintf("Date = %s\nInt = 1\n", dateStr), + }, + "array fields": { + input: struct { + IntArray0 [0]int + IntArray3 [3]int + }{[0]int{}, [3]int{1, 2, 3}}, + wantOutput: "IntArray0 = []\nIntArray3 = [1, 2, 3]\n", + }, + "slice fields": { + input: struct{ IntSliceNil, IntSlice0, IntSlice3 []int }{ + nil, []int{}, []int{1, 2, 3}, + }, + wantOutput: "IntSlice0 = []\nIntSlice3 = [1, 2, 3]\n", + }, + "datetime slices": { + input: struct{ DatetimeSlice []time.Time }{ + []time.Time{date, date}, + }, + wantOutput: fmt.Sprintf("DatetimeSlice = [%s, %s]\n", + dateStr, dateStr), + }, + "nested arrays and slices": { + input: struct { + SliceOfArrays [][2]int + ArrayOfSlices [2][]int + SliceOfArraysOfSlices [][2][]int + ArrayOfSlicesOfArrays [2][][2]int + SliceOfMixedArrays [][2]interface{} + ArrayOfMixedSlices [2][]interface{} + }{ + [][2]int{{1, 2}, {3, 4}}, + [2][]int{{1, 2}, {3, 4}}, + [][2][]int{ + { + {1, 2}, {3, 4}, + }, + { + {5, 6}, {7, 8}, + }, + }, + [2][][2]int{ + { + {1, 2}, {3, 4}, + }, + { + {5, 6}, {7, 8}, + }, + }, + [][2]interface{}{ + {1, 2}, {"a", "b"}, + }, + [2][]interface{}{ + {1, 2}, {"a", "b"}, + }, + }, + wantOutput: `SliceOfArrays = [[1, 2], [3, 4]] +ArrayOfSlices = [[1, 2], [3, 4]] +SliceOfArraysOfSlices = [[[1, 2], [3, 4]], [[5, 6], [7, 8]]] +ArrayOfSlicesOfArrays = [[[1, 2], [3, 4]], [[5, 6], [7, 8]]] +SliceOfMixedArrays = [[1, 2], ["a", "b"]] +ArrayOfMixedSlices = [[1, 2], ["a", "b"]] +`, + }, + "empty slice": { + input: struct{ Empty []interface{} }{[]interface{}{}}, + wantOutput: "Empty = []\n", + }, + "(error) slice with element type mismatch (string and integer)": { + input: struct{ Mixed []interface{} }{[]interface{}{1, "a"}}, + wantError: errArrayMixedElementTypes, + }, + "(error) slice with element type mismatch (integer and float)": { + input: struct{ Mixed []interface{} }{[]interface{}{1, 2.5}}, + wantError: errArrayMixedElementTypes, + }, + "slice with elems of differing Go types, same TOML types": { + input: struct { + MixedInts []interface{} + MixedFloats []interface{} + }{ + []interface{}{ + int(1), int8(2), int16(3), int32(4), int64(5), + uint(1), uint8(2), uint16(3), uint32(4), uint64(5), + }, + []interface{}{float32(1.5), float64(2.5)}, + }, + wantOutput: "MixedInts = [1, 2, 3, 4, 5, 1, 2, 3, 4, 5]\n" + + "MixedFloats = [1.5, 2.5]\n", + }, + "(error) slice w/ element type mismatch (one is nested array)": { + input: struct{ Mixed []interface{} }{ + []interface{}{1, []interface{}{2}}, + }, + wantError: errArrayMixedElementTypes, + }, + "(error) slice with 1 nil element": { + input: struct{ NilElement1 []interface{} }{[]interface{}{nil}}, + wantError: errArrayNilElement, + }, + "(error) slice with 1 nil element (and other non-nil elements)": { + input: struct{ NilElement []interface{} }{ + []interface{}{1, nil}, + }, + wantError: errArrayNilElement, + }, + "simple map": { + input: map[string]int{"a": 1, "b": 2}, + wantOutput: "a = 1\nb = 2\n", + }, + "map with interface{} value type": { + input: map[string]interface{}{"a": 1, "b": "c"}, + wantOutput: "a = 1\nb = \"c\"\n", + }, + "map with interface{} value type, some of which are structs": { + input: map[string]interface{}{ + "a": struct{ Int int }{2}, + "b": 1, + }, + wantOutput: "b = 1\n\n[a]\n Int = 2\n", + }, + "nested map": { + input: map[string]map[string]int{ + "a": {"b": 1}, + "c": {"d": 2}, + }, + wantOutput: "[a]\n b = 1\n\n[c]\n d = 2\n", + }, + "nested struct": { + input: struct{ Struct struct{ Int int } }{ + struct{ Int int }{1}, + }, + wantOutput: "[Struct]\n Int = 1\n", + }, + "nested struct and non-struct field": { + input: struct { + Struct struct{ Int int } + Bool bool + }{struct{ Int int }{1}, true}, + wantOutput: "Bool = true\n\n[Struct]\n Int = 1\n", + }, + "2 nested structs": { + input: struct{ Struct1, Struct2 struct{ Int int } }{ + struct{ Int int }{1}, struct{ Int int }{2}, + }, + wantOutput: "[Struct1]\n Int = 1\n\n[Struct2]\n Int = 2\n", + }, + "deeply nested structs": { + input: struct { + Struct1, Struct2 struct{ Struct3 *struct{ Int int } } + }{ + struct{ Struct3 *struct{ Int int } }{&struct{ Int int }{1}}, + struct{ Struct3 *struct{ Int int } }{nil}, + }, + wantOutput: "[Struct1]\n [Struct1.Struct3]\n Int = 1" + + "\n\n[Struct2]\n", + }, + "nested struct with nil struct elem": { + input: struct { + Struct struct{ Inner *struct{ Int int } } + }{ + struct{ Inner *struct{ Int int } }{nil}, + }, + wantOutput: "[Struct]\n", + }, + "nested struct with no fields": { + input: struct { + Struct struct{ Inner struct{} } + }{ + struct{ Inner struct{} }{struct{}{}}, + }, + wantOutput: "[Struct]\n [Struct.Inner]\n", + }, + "struct with tags": { + input: struct { + Struct struct { + Int int `toml:"_int"` + } `toml:"_struct"` + Bool bool `toml:"_bool"` + }{ + struct { + Int int `toml:"_int"` + }{1}, true, + }, + wantOutput: "_bool = true\n\n[_struct]\n _int = 1\n", + }, + "embedded struct": { + input: struct{ Embedded }{Embedded{1}}, + wantOutput: "_int = 1\n", + }, + "embedded *struct": { + input: struct{ *Embedded }{&Embedded{1}}, + wantOutput: "_int = 1\n", + }, + "nested embedded struct": { + input: struct { + Struct struct{ Embedded } `toml:"_struct"` + }{struct{ Embedded }{Embedded{1}}}, + wantOutput: "[_struct]\n _int = 1\n", + }, + "nested embedded *struct": { + input: struct { + Struct struct{ *Embedded } `toml:"_struct"` + }{struct{ *Embedded }{&Embedded{1}}}, + wantOutput: "[_struct]\n _int = 1\n", + }, + "array of tables": { + input: struct { + Structs []*struct{ Int int } `toml:"struct"` + }{ + []*struct{ Int int }{{1}, {3}}, + }, + wantOutput: "[[struct]]\n Int = 1\n\n[[struct]]\n Int = 3\n", + }, + "array of tables order": { + input: map[string]interface{}{ + "map": map[string]interface{}{ + "zero": 5, + "arr": []map[string]int{ + map[string]int{ + "friend": 5, + }, + }, + }, + }, + wantOutput: "[map]\n zero = 5\n\n [[map.arr]]\n friend = 5\n", + }, + "(error) top-level slice": { + input: []struct{ Int int }{{1}, {2}, {3}}, + wantError: errNoKey, + }, + "(error) slice of slice": { + input: struct { + Slices [][]struct{ Int int } + }{ + [][]struct{ Int int }{{{1}}, {{2}}, {{3}}}, + }, + wantError: errArrayNoTable, + }, + "(error) map no string key": { + input: map[int]string{1: ""}, + wantError: errNonString, + }, + "(error) anonymous non-struct": { + input: struct{ NonStruct }{5}, + wantError: errAnonNonStruct, + }, + "(error) empty key name": { + input: map[string]int{"": 1}, + wantError: errAnything, + }, + "(error) empty map name": { + input: map[string]interface{}{ + "": map[string]int{"v": 1}, + }, + wantError: errAnything, + }, + } + for label, test := range tests { + encodeExpected(t, label, test.input, test.wantOutput, test.wantError) + } +} + +func TestEncodeNestedTableArrays(t *testing.T) { + type song struct { + Name string `toml:"name"` + } + type album struct { + Name string `toml:"name"` + Songs []song `toml:"songs"` + } + type springsteen struct { + Albums []album `toml:"albums"` + } + value := springsteen{ + []album{ + {"Born to Run", + []song{{"Jungleland"}, {"Meeting Across the River"}}}, + {"Born in the USA", + []song{{"Glory Days"}, {"Dancing in the Dark"}}}, + }, + } + expected := `[[albums]] + name = "Born to Run" + + [[albums.songs]] + name = "Jungleland" + + [[albums.songs]] + name = "Meeting Across the River" + +[[albums]] + name = "Born in the USA" + + [[albums.songs]] + name = "Glory Days" + + [[albums.songs]] + name = "Dancing in the Dark" +` + encodeExpected(t, "nested table arrays", value, expected, nil) +} + +func TestEncodeArrayHashWithNormalHashOrder(t *testing.T) { + type Alpha struct { + V int + } + type Beta struct { + V int + } + type Conf struct { + V int + A Alpha + B []Beta + } + + val := Conf{ + V: 1, + A: Alpha{2}, + B: []Beta{{3}}, + } + expected := "V = 1\n\n[A]\n V = 2\n\n[[B]]\n V = 3\n" + encodeExpected(t, "array hash with normal hash order", val, expected, nil) +} + +func encodeExpected( + t *testing.T, label string, val interface{}, wantStr string, wantErr error, +) { + var buf bytes.Buffer + enc := NewEncoder(&buf) + err := enc.Encode(val) + if err != wantErr { + if wantErr != nil { + if wantErr == errAnything && err != nil { + return + } + t.Errorf("%s: want Encode error %v, got %v", label, wantErr, err) + } else { + t.Errorf("%s: Encode failed: %s", label, err) + } + } + if err != nil { + return + } + if got := buf.String(); wantStr != got { + t.Errorf("%s: want\n-----\n%q\n-----\nbut got\n-----\n%q\n-----\n", + label, wantStr, got) + } +} + +func ExampleEncoder_Encode() { + date, _ := time.Parse(time.RFC822, "14 Mar 10 18:00 UTC") + var config = map[string]interface{}{ + "date": date, + "counts": []int{1, 1, 2, 3, 5, 8}, + "hash": map[string]string{ + "key1": "val1", + "key2": "val2", + }, + } + buf := new(bytes.Buffer) + if err := NewEncoder(buf).Encode(config); err != nil { + log.Fatal(err) + } + fmt.Println(buf.String()) + + // Output: + // counts = [1, 1, 2, 3, 5, 8] + // date = 2010-03-14T18:00:00Z + // + // [hash] + // key1 = "val1" + // key2 = "val2" +} diff --git a/Godeps/_workspace/src/github.com/BurntSushi/toml/encoding_types.go b/Godeps/_workspace/src/github.com/BurntSushi/toml/encoding_types.go new file mode 100644 index 000000000..d36e1dd60 --- /dev/null +++ b/Godeps/_workspace/src/github.com/BurntSushi/toml/encoding_types.go @@ -0,0 +1,19 @@ +// +build go1.2 + +package toml + +// In order to support Go 1.1, we define our own TextMarshaler and +// TextUnmarshaler types. For Go 1.2+, we just alias them with the +// standard library interfaces. + +import ( + "encoding" +) + +// TextMarshaler is a synonym for encoding.TextMarshaler. It is defined here +// so that Go 1.1 can be supported. +type TextMarshaler encoding.TextMarshaler + +// TextUnmarshaler is a synonym for encoding.TextUnmarshaler. It is defined +// here so that Go 1.1 can be supported. +type TextUnmarshaler encoding.TextUnmarshaler diff --git a/Godeps/_workspace/src/github.com/BurntSushi/toml/encoding_types_1.1.go b/Godeps/_workspace/src/github.com/BurntSushi/toml/encoding_types_1.1.go new file mode 100644 index 000000000..e8d503d04 --- /dev/null +++ b/Godeps/_workspace/src/github.com/BurntSushi/toml/encoding_types_1.1.go @@ -0,0 +1,18 @@ +// +build !go1.2 + +package toml + +// These interfaces were introduced in Go 1.2, so we add them manually when +// compiling for Go 1.1. + +// TextMarshaler is a synonym for encoding.TextMarshaler. It is defined here +// so that Go 1.1 can be supported. +type TextMarshaler interface { + MarshalText() (text []byte, err error) +} + +// TextUnmarshaler is a synonym for encoding.TextUnmarshaler. It is defined +// here so that Go 1.1 can be supported. +type TextUnmarshaler interface { + UnmarshalText(text []byte) error +} diff --git a/Godeps/_workspace/src/github.com/BurntSushi/toml/lex.go b/Godeps/_workspace/src/github.com/BurntSushi/toml/lex.go new file mode 100644 index 000000000..219122857 --- /dev/null +++ b/Godeps/_workspace/src/github.com/BurntSushi/toml/lex.go @@ -0,0 +1,874 @@ +package toml + +import ( + "fmt" + "strings" + "unicode/utf8" +) + +type itemType int + +const ( + itemError itemType = iota + itemNIL // used in the parser to indicate no type + itemEOF + itemText + itemString + itemRawString + itemMultilineString + itemRawMultilineString + itemBool + itemInteger + itemFloat + itemDatetime + itemArray // the start of an array + itemArrayEnd + itemTableStart + itemTableEnd + itemArrayTableStart + itemArrayTableEnd + itemKeyStart + itemCommentStart +) + +const ( + eof = 0 + tableStart = '[' + tableEnd = ']' + arrayTableStart = '[' + arrayTableEnd = ']' + tableSep = '.' + keySep = '=' + arrayStart = '[' + arrayEnd = ']' + arrayValTerm = ',' + commentStart = '#' + stringStart = '"' + stringEnd = '"' + rawStringStart = '\'' + rawStringEnd = '\'' +) + +type stateFn func(lx *lexer) stateFn + +type lexer struct { + input string + start int + pos int + width int + line int + state stateFn + items chan item + + // A stack of state functions used to maintain context. + // The idea is to reuse parts of the state machine in various places. + // For example, values can appear at the top level or within arbitrarily + // nested arrays. The last state on the stack is used after a value has + // been lexed. Similarly for comments. + stack []stateFn +} + +type item struct { + typ itemType + val string + line int +} + +func (lx *lexer) nextItem() item { + for { + select { + case item := <-lx.items: + return item + default: + lx.state = lx.state(lx) + } + } +} + +func lex(input string) *lexer { + lx := &lexer{ + input: input + "\n", + state: lexTop, + line: 1, + items: make(chan item, 10), + stack: make([]stateFn, 0, 10), + } + return lx +} + +func (lx *lexer) push(state stateFn) { + lx.stack = append(lx.stack, state) +} + +func (lx *lexer) pop() stateFn { + if len(lx.stack) == 0 { + return lx.errorf("BUG in lexer: no states to pop.") + } + last := lx.stack[len(lx.stack)-1] + lx.stack = lx.stack[0 : len(lx.stack)-1] + return last +} + +func (lx *lexer) current() string { + return lx.input[lx.start:lx.pos] +} + +func (lx *lexer) emit(typ itemType) { + lx.items <- item{typ, lx.current(), lx.line} + lx.start = lx.pos +} + +func (lx *lexer) emitTrim(typ itemType) { + lx.items <- item{typ, strings.TrimSpace(lx.current()), lx.line} + lx.start = lx.pos +} + +func (lx *lexer) next() (r rune) { + if lx.pos >= len(lx.input) { + lx.width = 0 + return eof + } + + if lx.input[lx.pos] == '\n' { + lx.line++ + } + r, lx.width = utf8.DecodeRuneInString(lx.input[lx.pos:]) + lx.pos += lx.width + return r +} + +// ignore skips over the pending input before this point. +func (lx *lexer) ignore() { + lx.start = lx.pos +} + +// backup steps back one rune. Can be called only once per call of next. +func (lx *lexer) backup() { + lx.pos -= lx.width + if lx.pos < len(lx.input) && lx.input[lx.pos] == '\n' { + lx.line-- + } +} + +// accept consumes the next rune if it's equal to `valid`. +func (lx *lexer) accept(valid rune) bool { + if lx.next() == valid { + return true + } + lx.backup() + return false +} + +// peek returns but does not consume the next rune in the input. +func (lx *lexer) peek() rune { + r := lx.next() + lx.backup() + return r +} + +// errorf stops all lexing by emitting an error and returning `nil`. +// Note that any value that is a character is escaped if it's a special +// character (new lines, tabs, etc.). +func (lx *lexer) errorf(format string, values ...interface{}) stateFn { + lx.items <- item{ + itemError, + fmt.Sprintf(format, values...), + lx.line, + } + return nil +} + +// lexTop consumes elements at the top level of TOML data. +func lexTop(lx *lexer) stateFn { + r := lx.next() + if isWhitespace(r) || isNL(r) { + return lexSkip(lx, lexTop) + } + + switch r { + case commentStart: + lx.push(lexTop) + return lexCommentStart + case tableStart: + return lexTableStart + case eof: + if lx.pos > lx.start { + return lx.errorf("Unexpected EOF.") + } + lx.emit(itemEOF) + return nil + } + + // At this point, the only valid item can be a key, so we back up + // and let the key lexer do the rest. + lx.backup() + lx.push(lexTopEnd) + return lexKeyStart +} + +// lexTopEnd is entered whenever a top-level item has been consumed. (A value +// or a table.) It must see only whitespace, and will turn back to lexTop +// upon a new line. If it sees EOF, it will quit the lexer successfully. +func lexTopEnd(lx *lexer) stateFn { + r := lx.next() + switch { + case r == commentStart: + // a comment will read to a new line for us. + lx.push(lexTop) + return lexCommentStart + case isWhitespace(r): + return lexTopEnd + case isNL(r): + lx.ignore() + return lexTop + case r == eof: + lx.ignore() + return lexTop + } + return lx.errorf("Expected a top-level item to end with a new line, "+ + "comment or EOF, but got %q instead.", r) +} + +// lexTable lexes the beginning of a table. Namely, it makes sure that +// it starts with a character other than '.' and ']'. +// It assumes that '[' has already been consumed. +// It also handles the case that this is an item in an array of tables. +// e.g., '[[name]]'. +func lexTableStart(lx *lexer) stateFn { + if lx.peek() == arrayTableStart { + lx.next() + lx.emit(itemArrayTableStart) + lx.push(lexArrayTableEnd) + } else { + lx.emit(itemTableStart) + lx.push(lexTableEnd) + } + return lexTableNameStart +} + +func lexTableEnd(lx *lexer) stateFn { + lx.emit(itemTableEnd) + return lexTopEnd +} + +func lexArrayTableEnd(lx *lexer) stateFn { + if r := lx.next(); r != arrayTableEnd { + return lx.errorf("Expected end of table array name delimiter %q, "+ + "but got %q instead.", arrayTableEnd, r) + } + lx.emit(itemArrayTableEnd) + return lexTopEnd +} + +func lexTableNameStart(lx *lexer) stateFn { + switch r := lx.peek(); { + case r == tableEnd || r == eof: + return lx.errorf("Unexpected end of table name. (Table names cannot " + + "be empty.)") + case r == tableSep: + return lx.errorf("Unexpected table separator. (Table names cannot " + + "be empty.)") + case r == stringStart || r == rawStringStart: + lx.ignore() + lx.push(lexTableNameEnd) + return lexValue // reuse string lexing + case isWhitespace(r): + return lexTableNameStart + default: + return lexBareTableName + } +} + +// lexTableName lexes the name of a table. It assumes that at least one +// valid character for the table has already been read. +func lexBareTableName(lx *lexer) stateFn { + switch r := lx.next(); { + case isBareKeyChar(r): + return lexBareTableName + case r == tableSep || r == tableEnd: + lx.backup() + lx.emitTrim(itemText) + return lexTableNameEnd + default: + return lx.errorf("Bare keys cannot contain %q.", r) + } +} + +// lexTableNameEnd reads the end of a piece of a table name, optionally +// consuming whitespace. +func lexTableNameEnd(lx *lexer) stateFn { + switch r := lx.next(); { + case isWhitespace(r): + return lexTableNameEnd + case r == tableSep: + lx.ignore() + return lexTableNameStart + case r == tableEnd: + return lx.pop() + default: + return lx.errorf("Expected '.' or ']' to end table name, but got %q "+ + "instead.", r) + } +} + +// lexKeyStart consumes a key name up until the first non-whitespace character. +// lexKeyStart will ignore whitespace. +func lexKeyStart(lx *lexer) stateFn { + r := lx.peek() + switch { + case r == keySep: + return lx.errorf("Unexpected key separator %q.", keySep) + case isWhitespace(r) || isNL(r): + lx.next() + return lexSkip(lx, lexKeyStart) + case r == stringStart || r == rawStringStart: + lx.ignore() + lx.emit(itemKeyStart) + lx.push(lexKeyEnd) + return lexValue // reuse string lexing + default: + lx.ignore() + lx.emit(itemKeyStart) + return lexBareKey + } +} + +// lexBareKey consumes the text of a bare key. Assumes that the first character +// (which is not whitespace) has not yet been consumed. +func lexBareKey(lx *lexer) stateFn { + switch r := lx.next(); { + case isBareKeyChar(r): + return lexBareKey + case isWhitespace(r): + lx.emitTrim(itemText) + return lexKeyEnd + case r == keySep: + lx.backup() + lx.emitTrim(itemText) + return lexKeyEnd + default: + return lx.errorf("Bare keys cannot contain %q.", r) + } +} + +// lexKeyEnd consumes the end of a key and trims whitespace (up to the key +// separator). +func lexKeyEnd(lx *lexer) stateFn { + switch r := lx.next(); { + case r == keySep: + return lexSkip(lx, lexValue) + case isWhitespace(r): + return lexSkip(lx, lexKeyEnd) + default: + return lx.errorf("Expected key separator %q, but got %q instead.", + keySep, r) + } +} + +// lexValue starts the consumption of a value anywhere a value is expected. +// lexValue will ignore whitespace. +// After a value is lexed, the last state on the next is popped and returned. +func lexValue(lx *lexer) stateFn { + // We allow whitespace to precede a value, but NOT new lines. + // In array syntax, the array states are responsible for ignoring new + // lines. + r := lx.next() + if isWhitespace(r) { + return lexSkip(lx, lexValue) + } + + switch { + case r == arrayStart: + lx.ignore() + lx.emit(itemArray) + return lexArrayValue + case r == stringStart: + if lx.accept(stringStart) { + if lx.accept(stringStart) { + lx.ignore() // Ignore """ + return lexMultilineString + } + lx.backup() + } + lx.ignore() // ignore the '"' + return lexString + case r == rawStringStart: + if lx.accept(rawStringStart) { + if lx.accept(rawStringStart) { + lx.ignore() // Ignore """ + return lexMultilineRawString + } + lx.backup() + } + lx.ignore() // ignore the "'" + return lexRawString + case r == 't': + return lexTrue + case r == 'f': + return lexFalse + case r == '-': + return lexNumberStart + case isDigit(r): + lx.backup() // avoid an extra state and use the same as above + return lexNumberOrDateStart + case r == '.': // special error case, be kind to users + return lx.errorf("Floats must start with a digit, not '.'.") + } + return lx.errorf("Expected value but found %q instead.", r) +} + +// lexArrayValue consumes one value in an array. It assumes that '[' or ',' +// have already been consumed. All whitespace and new lines are ignored. +func lexArrayValue(lx *lexer) stateFn { + r := lx.next() + switch { + case isWhitespace(r) || isNL(r): + return lexSkip(lx, lexArrayValue) + case r == commentStart: + lx.push(lexArrayValue) + return lexCommentStart + case r == arrayValTerm: + return lx.errorf("Unexpected array value terminator %q.", + arrayValTerm) + case r == arrayEnd: + return lexArrayEnd + } + + lx.backup() + lx.push(lexArrayValueEnd) + return lexValue +} + +// lexArrayValueEnd consumes the cruft between values of an array. Namely, +// it ignores whitespace and expects either a ',' or a ']'. +func lexArrayValueEnd(lx *lexer) stateFn { + r := lx.next() + switch { + case isWhitespace(r) || isNL(r): + return lexSkip(lx, lexArrayValueEnd) + case r == commentStart: + lx.push(lexArrayValueEnd) + return lexCommentStart + case r == arrayValTerm: + lx.ignore() + return lexArrayValue // move on to the next value + case r == arrayEnd: + return lexArrayEnd + } + return lx.errorf("Expected an array value terminator %q or an array "+ + "terminator %q, but got %q instead.", arrayValTerm, arrayEnd, r) +} + +// lexArrayEnd finishes the lexing of an array. It assumes that a ']' has +// just been consumed. +func lexArrayEnd(lx *lexer) stateFn { + lx.ignore() + lx.emit(itemArrayEnd) + return lx.pop() +} + +// lexString consumes the inner contents of a string. It assumes that the +// beginning '"' has already been consumed and ignored. +func lexString(lx *lexer) stateFn { + r := lx.next() + switch { + case isNL(r): + return lx.errorf("Strings cannot contain new lines.") + case r == '\\': + lx.push(lexString) + return lexStringEscape + case r == stringEnd: + lx.backup() + lx.emit(itemString) + lx.next() + lx.ignore() + return lx.pop() + } + return lexString +} + +// lexMultilineString consumes the inner contents of a string. It assumes that +// the beginning '"""' has already been consumed and ignored. +func lexMultilineString(lx *lexer) stateFn { + r := lx.next() + switch { + case r == '\\': + return lexMultilineStringEscape + case r == stringEnd: + if lx.accept(stringEnd) { + if lx.accept(stringEnd) { + lx.backup() + lx.backup() + lx.backup() + lx.emit(itemMultilineString) + lx.next() + lx.next() + lx.next() + lx.ignore() + return lx.pop() + } + lx.backup() + } + } + return lexMultilineString +} + +// lexRawString consumes a raw string. Nothing can be escaped in such a string. +// It assumes that the beginning "'" has already been consumed and ignored. +func lexRawString(lx *lexer) stateFn { + r := lx.next() + switch { + case isNL(r): + return lx.errorf("Strings cannot contain new lines.") + case r == rawStringEnd: + lx.backup() + lx.emit(itemRawString) + lx.next() + lx.ignore() + return lx.pop() + } + return lexRawString +} + +// lexMultilineRawString consumes a raw string. Nothing can be escaped in such +// a string. It assumes that the beginning "'" has already been consumed and +// ignored. +func lexMultilineRawString(lx *lexer) stateFn { + r := lx.next() + switch { + case r == rawStringEnd: + if lx.accept(rawStringEnd) { + if lx.accept(rawStringEnd) { + lx.backup() + lx.backup() + lx.backup() + lx.emit(itemRawMultilineString) + lx.next() + lx.next() + lx.next() + lx.ignore() + return lx.pop() + } + lx.backup() + } + } + return lexMultilineRawString +} + +// lexMultilineStringEscape consumes an escaped character. It assumes that the +// preceding '\\' has already been consumed. +func lexMultilineStringEscape(lx *lexer) stateFn { + // Handle the special case first: + if isNL(lx.next()) { + lx.next() + return lexMultilineString + } else { + lx.backup() + lx.push(lexMultilineString) + return lexStringEscape(lx) + } +} + +func lexStringEscape(lx *lexer) stateFn { + r := lx.next() + switch r { + case 'b': + fallthrough + case 't': + fallthrough + case 'n': + fallthrough + case 'f': + fallthrough + case 'r': + fallthrough + case '"': + fallthrough + case '\\': + return lx.pop() + case 'u': + return lexShortUnicodeEscape + case 'U': + return lexLongUnicodeEscape + } + return lx.errorf("Invalid escape character %q. Only the following "+ + "escape characters are allowed: "+ + "\\b, \\t, \\n, \\f, \\r, \\\", \\/, \\\\, "+ + "\\uXXXX and \\UXXXXXXXX.", r) +} + +func lexShortUnicodeEscape(lx *lexer) stateFn { + var r rune + for i := 0; i < 4; i++ { + r = lx.next() + if !isHexadecimal(r) { + return lx.errorf("Expected four hexadecimal digits after '\\u', "+ + "but got '%s' instead.", lx.current()) + } + } + return lx.pop() +} + +func lexLongUnicodeEscape(lx *lexer) stateFn { + var r rune + for i := 0; i < 8; i++ { + r = lx.next() + if !isHexadecimal(r) { + return lx.errorf("Expected eight hexadecimal digits after '\\U', "+ + "but got '%s' instead.", lx.current()) + } + } + return lx.pop() +} + +// lexNumberOrDateStart consumes either a (positive) integer, float or +// datetime. It assumes that NO negative sign has been consumed. +func lexNumberOrDateStart(lx *lexer) stateFn { + r := lx.next() + if !isDigit(r) { + if r == '.' { + return lx.errorf("Floats must start with a digit, not '.'.") + } else { + return lx.errorf("Expected a digit but got %q.", r) + } + } + return lexNumberOrDate +} + +// lexNumberOrDate consumes either a (positive) integer, float or datetime. +func lexNumberOrDate(lx *lexer) stateFn { + r := lx.next() + switch { + case r == '-': + if lx.pos-lx.start != 5 { + return lx.errorf("All ISO8601 dates must be in full Zulu form.") + } + return lexDateAfterYear + case isDigit(r): + return lexNumberOrDate + case r == '.': + return lexFloatStart + } + + lx.backup() + lx.emit(itemInteger) + return lx.pop() +} + +// lexDateAfterYear consumes a full Zulu Datetime in ISO8601 format. +// It assumes that "YYYY-" has already been consumed. +func lexDateAfterYear(lx *lexer) stateFn { + formats := []rune{ + // digits are '0'. + // everything else is direct equality. + '0', '0', '-', '0', '0', + 'T', + '0', '0', ':', '0', '0', ':', '0', '0', + 'Z', + } + for _, f := range formats { + r := lx.next() + if f == '0' { + if !isDigit(r) { + return lx.errorf("Expected digit in ISO8601 datetime, "+ + "but found %q instead.", r) + } + } else if f != r { + return lx.errorf("Expected %q in ISO8601 datetime, "+ + "but found %q instead.", f, r) + } + } + lx.emit(itemDatetime) + return lx.pop() +} + +// lexNumberStart consumes either an integer or a float. It assumes that +// a negative sign has already been read, but that *no* digits have been +// consumed. lexNumberStart will move to the appropriate integer or float +// states. +func lexNumberStart(lx *lexer) stateFn { + // we MUST see a digit. Even floats have to start with a digit. + r := lx.next() + if !isDigit(r) { + if r == '.' { + return lx.errorf("Floats must start with a digit, not '.'.") + } else { + return lx.errorf("Expected a digit but got %q.", r) + } + } + return lexNumber +} + +// lexNumber consumes an integer or a float after seeing the first digit. +func lexNumber(lx *lexer) stateFn { + r := lx.next() + switch { + case isDigit(r): + return lexNumber + case r == '.': + return lexFloatStart + } + + lx.backup() + lx.emit(itemInteger) + return lx.pop() +} + +// lexFloatStart starts the consumption of digits of a float after a '.'. +// Namely, at least one digit is required. +func lexFloatStart(lx *lexer) stateFn { + r := lx.next() + if !isDigit(r) { + return lx.errorf("Floats must have a digit after the '.', but got "+ + "%q instead.", r) + } + return lexFloat +} + +// lexFloat consumes the digits of a float after a '.'. +// Assumes that one digit has been consumed after a '.' already. +func lexFloat(lx *lexer) stateFn { + r := lx.next() + if isDigit(r) { + return lexFloat + } + + lx.backup() + lx.emit(itemFloat) + return lx.pop() +} + +// lexConst consumes the s[1:] in s. It assumes that s[0] has already been +// consumed. +func lexConst(lx *lexer, s string) stateFn { + for i := range s[1:] { + if r := lx.next(); r != rune(s[i+1]) { + return lx.errorf("Expected %q, but found %q instead.", s[:i+1], + s[:i]+string(r)) + } + } + return nil +} + +// lexTrue consumes the "rue" in "true". It assumes that 't' has already +// been consumed. +func lexTrue(lx *lexer) stateFn { + if fn := lexConst(lx, "true"); fn != nil { + return fn + } + lx.emit(itemBool) + return lx.pop() +} + +// lexFalse consumes the "alse" in "false". It assumes that 'f' has already +// been consumed. +func lexFalse(lx *lexer) stateFn { + if fn := lexConst(lx, "false"); fn != nil { + return fn + } + lx.emit(itemBool) + return lx.pop() +} + +// lexCommentStart begins the lexing of a comment. It will emit +// itemCommentStart and consume no characters, passing control to lexComment. +func lexCommentStart(lx *lexer) stateFn { + lx.ignore() + lx.emit(itemCommentStart) + return lexComment +} + +// lexComment lexes an entire comment. It assumes that '#' has been consumed. +// It will consume *up to* the first new line character, and pass control +// back to the last state on the stack. +func lexComment(lx *lexer) stateFn { + r := lx.peek() + if isNL(r) || r == eof { + lx.emit(itemText) + return lx.pop() + } + lx.next() + return lexComment +} + +// lexSkip ignores all slurped input and moves on to the next state. +func lexSkip(lx *lexer, nextState stateFn) stateFn { + return func(lx *lexer) stateFn { + lx.ignore() + return nextState + } +} + +// isWhitespace returns true if `r` is a whitespace character according +// to the spec. +func isWhitespace(r rune) bool { + return r == '\t' || r == ' ' +} + +func isNL(r rune) bool { + return r == '\n' || r == '\r' +} + +func isDigit(r rune) bool { + return r >= '0' && r <= '9' +} + +func isHexadecimal(r rune) bool { + return (r >= '0' && r <= '9') || + (r >= 'a' && r <= 'f') || + (r >= 'A' && r <= 'F') +} + +func isBareKeyChar(r rune) bool { + return (r >= 'A' && r <= 'Z') || + (r >= 'a' && r <= 'z') || + (r >= '0' && r <= '9') || + r == '_' || + r == '-' +} + +func (itype itemType) String() string { + switch itype { + case itemError: + return "Error" + case itemNIL: + return "NIL" + case itemEOF: + return "EOF" + case itemText: + return "Text" + case itemString: + return "String" + case itemRawString: + return "String" + case itemMultilineString: + return "String" + case itemRawMultilineString: + return "String" + case itemBool: + return "Bool" + case itemInteger: + return "Integer" + case itemFloat: + return "Float" + case itemDatetime: + return "DateTime" + case itemTableStart: + return "TableStart" + case itemTableEnd: + return "TableEnd" + case itemKeyStart: + return "KeyStart" + case itemArray: + return "Array" + case itemArrayEnd: + return "ArrayEnd" + case itemCommentStart: + return "CommentStart" + } + panic(fmt.Sprintf("BUG: Unknown type '%d'.", int(itype))) +} + +func (item item) String() string { + return fmt.Sprintf("(%s, %s)", item.typ.String(), item.val) +} diff --git a/Godeps/_workspace/src/github.com/BurntSushi/toml/parse.go b/Godeps/_workspace/src/github.com/BurntSushi/toml/parse.go new file mode 100644 index 000000000..c6069be1f --- /dev/null +++ b/Godeps/_workspace/src/github.com/BurntSushi/toml/parse.go @@ -0,0 +1,498 @@ +package toml + +import ( + "fmt" + "log" + "strconv" + "strings" + "time" + "unicode" + "unicode/utf8" +) + +type parser struct { + mapping map[string]interface{} + types map[string]tomlType + lx *lexer + + // A list of keys in the order that they appear in the TOML data. + ordered []Key + + // the full key for the current hash in scope + context Key + + // the base key name for everything except hashes + currentKey string + + // rough approximation of line number + approxLine int + + // A map of 'key.group.names' to whether they were created implicitly. + implicits map[string]bool +} + +type parseError string + +func (pe parseError) Error() string { + return string(pe) +} + +func parse(data string) (p *parser, err error) { + defer func() { + if r := recover(); r != nil { + var ok bool + if err, ok = r.(parseError); ok { + return + } + panic(r) + } + }() + + p = &parser{ + mapping: make(map[string]interface{}), + types: make(map[string]tomlType), + lx: lex(data), + ordered: make([]Key, 0), + implicits: make(map[string]bool), + } + for { + item := p.next() + if item.typ == itemEOF { + break + } + p.topLevel(item) + } + + return p, nil +} + +func (p *parser) panicf(format string, v ...interface{}) { + msg := fmt.Sprintf("Near line %d (last key parsed '%s'): %s", + p.approxLine, p.current(), fmt.Sprintf(format, v...)) + panic(parseError(msg)) +} + +func (p *parser) next() item { + it := p.lx.nextItem() + if it.typ == itemError { + p.panicf("%s", it.val) + } + return it +} + +func (p *parser) bug(format string, v ...interface{}) { + log.Fatalf("BUG: %s\n\n", fmt.Sprintf(format, v...)) +} + +func (p *parser) expect(typ itemType) item { + it := p.next() + p.assertEqual(typ, it.typ) + return it +} + +func (p *parser) assertEqual(expected, got itemType) { + if expected != got { + p.bug("Expected '%s' but got '%s'.", expected, got) + } +} + +func (p *parser) topLevel(item item) { + switch item.typ { + case itemCommentStart: + p.approxLine = item.line + p.expect(itemText) + case itemTableStart: + kg := p.next() + p.approxLine = kg.line + + var key Key + for ; kg.typ != itemTableEnd && kg.typ != itemEOF; kg = p.next() { + key = append(key, p.keyString(kg)) + } + p.assertEqual(itemTableEnd, kg.typ) + + p.establishContext(key, false) + p.setType("", tomlHash) + p.ordered = append(p.ordered, key) + case itemArrayTableStart: + kg := p.next() + p.approxLine = kg.line + + var key Key + for ; kg.typ != itemArrayTableEnd && kg.typ != itemEOF; kg = p.next() { + key = append(key, p.keyString(kg)) + } + p.assertEqual(itemArrayTableEnd, kg.typ) + + p.establishContext(key, true) + p.setType("", tomlArrayHash) + p.ordered = append(p.ordered, key) + case itemKeyStart: + kname := p.next() + p.approxLine = kname.line + p.currentKey = p.keyString(kname) + + val, typ := p.value(p.next()) + p.setValue(p.currentKey, val) + p.setType(p.currentKey, typ) + p.ordered = append(p.ordered, p.context.add(p.currentKey)) + p.currentKey = "" + default: + p.bug("Unexpected type at top level: %s", item.typ) + } +} + +// Gets a string for a key (or part of a key in a table name). +func (p *parser) keyString(it item) string { + switch it.typ { + case itemText: + return it.val + case itemString, itemMultilineString, + itemRawString, itemRawMultilineString: + s, _ := p.value(it) + return s.(string) + default: + p.bug("Unexpected key type: %s", it.typ) + panic("unreachable") + } +} + +// value translates an expected value from the lexer into a Go value wrapped +// as an empty interface. +func (p *parser) value(it item) (interface{}, tomlType) { + switch it.typ { + case itemString: + return p.replaceEscapes(it.val), p.typeOfPrimitive(it) + case itemMultilineString: + trimmed := stripFirstNewline(stripEscapedWhitespace(it.val)) + return p.replaceEscapes(trimmed), p.typeOfPrimitive(it) + case itemRawString: + return it.val, p.typeOfPrimitive(it) + case itemRawMultilineString: + return stripFirstNewline(it.val), p.typeOfPrimitive(it) + case itemBool: + switch it.val { + case "true": + return true, p.typeOfPrimitive(it) + case "false": + return false, p.typeOfPrimitive(it) + } + p.bug("Expected boolean value, but got '%s'.", it.val) + case itemInteger: + num, err := strconv.ParseInt(it.val, 10, 64) + if err != nil { + // See comment below for floats describing why we make a + // distinction between a bug and a user error. + if e, ok := err.(*strconv.NumError); ok && + e.Err == strconv.ErrRange { + + p.panicf("Integer '%s' is out of the range of 64-bit "+ + "signed integers.", it.val) + } else { + p.bug("Expected integer value, but got '%s'.", it.val) + } + } + return num, p.typeOfPrimitive(it) + case itemFloat: + num, err := strconv.ParseFloat(it.val, 64) + if err != nil { + // Distinguish float values. Normally, it'd be a bug if the lexer + // provides an invalid float, but it's possible that the float is + // out of range of valid values (which the lexer cannot determine). + // So mark the former as a bug but the latter as a legitimate user + // error. + // + // This is also true for integers. + if e, ok := err.(*strconv.NumError); ok && + e.Err == strconv.ErrRange { + + p.panicf("Float '%s' is out of the range of 64-bit "+ + "IEEE-754 floating-point numbers.", it.val) + } else { + p.bug("Expected float value, but got '%s'.", it.val) + } + } + return num, p.typeOfPrimitive(it) + case itemDatetime: + t, err := time.Parse("2006-01-02T15:04:05Z", it.val) + if err != nil { + p.bug("Expected Zulu formatted DateTime, but got '%s'.", it.val) + } + return t, p.typeOfPrimitive(it) + case itemArray: + array := make([]interface{}, 0) + types := make([]tomlType, 0) + + for it = p.next(); it.typ != itemArrayEnd; it = p.next() { + if it.typ == itemCommentStart { + p.expect(itemText) + continue + } + + val, typ := p.value(it) + array = append(array, val) + types = append(types, typ) + } + return array, p.typeOfArray(types) + } + p.bug("Unexpected value type: %s", it.typ) + panic("unreachable") +} + +// establishContext sets the current context of the parser, +// where the context is either a hash or an array of hashes. Which one is +// set depends on the value of the `array` parameter. +// +// Establishing the context also makes sure that the key isn't a duplicate, and +// will create implicit hashes automatically. +func (p *parser) establishContext(key Key, array bool) { + var ok bool + + // Always start at the top level and drill down for our context. + hashContext := p.mapping + keyContext := make(Key, 0) + + // We only need implicit hashes for key[0:-1] + for _, k := range key[0 : len(key)-1] { + _, ok = hashContext[k] + keyContext = append(keyContext, k) + + // No key? Make an implicit hash and move on. + if !ok { + p.addImplicit(keyContext) + hashContext[k] = make(map[string]interface{}) + } + + // If the hash context is actually an array of tables, then set + // the hash context to the last element in that array. + // + // Otherwise, it better be a table, since this MUST be a key group (by + // virtue of it not being the last element in a key). + switch t := hashContext[k].(type) { + case []map[string]interface{}: + hashContext = t[len(t)-1] + case map[string]interface{}: + hashContext = t + default: + p.panicf("Key '%s' was already created as a hash.", keyContext) + } + } + + p.context = keyContext + if array { + // If this is the first element for this array, then allocate a new + // list of tables for it. + k := key[len(key)-1] + if _, ok := hashContext[k]; !ok { + hashContext[k] = make([]map[string]interface{}, 0, 5) + } + + // Add a new table. But make sure the key hasn't already been used + // for something else. + if hash, ok := hashContext[k].([]map[string]interface{}); ok { + hashContext[k] = append(hash, make(map[string]interface{})) + } else { + p.panicf("Key '%s' was already created and cannot be used as "+ + "an array.", keyContext) + } + } else { + p.setValue(key[len(key)-1], make(map[string]interface{})) + } + p.context = append(p.context, key[len(key)-1]) +} + +// setValue sets the given key to the given value in the current context. +// It will make sure that the key hasn't already been defined, account for +// implicit key groups. +func (p *parser) setValue(key string, value interface{}) { + var tmpHash interface{} + var ok bool + + hash := p.mapping + keyContext := make(Key, 0) + for _, k := range p.context { + keyContext = append(keyContext, k) + if tmpHash, ok = hash[k]; !ok { + p.bug("Context for key '%s' has not been established.", keyContext) + } + switch t := tmpHash.(type) { + case []map[string]interface{}: + // The context is a table of hashes. Pick the most recent table + // defined as the current hash. + hash = t[len(t)-1] + case map[string]interface{}: + hash = t + default: + p.bug("Expected hash to have type 'map[string]interface{}', but "+ + "it has '%T' instead.", tmpHash) + } + } + keyContext = append(keyContext, key) + + if _, ok := hash[key]; ok { + // Typically, if the given key has already been set, then we have + // to raise an error since duplicate keys are disallowed. However, + // it's possible that a key was previously defined implicitly. In this + // case, it is allowed to be redefined concretely. (See the + // `tests/valid/implicit-and-explicit-after.toml` test in `toml-test`.) + // + // But we have to make sure to stop marking it as an implicit. (So that + // another redefinition provokes an error.) + // + // Note that since it has already been defined (as a hash), we don't + // want to overwrite it. So our business is done. + if p.isImplicit(keyContext) { + p.removeImplicit(keyContext) + return + } + + // Otherwise, we have a concrete key trying to override a previous + // key, which is *always* wrong. + p.panicf("Key '%s' has already been defined.", keyContext) + } + hash[key] = value +} + +// setType sets the type of a particular value at a given key. +// It should be called immediately AFTER setValue. +// +// Note that if `key` is empty, then the type given will be applied to the +// current context (which is either a table or an array of tables). +func (p *parser) setType(key string, typ tomlType) { + keyContext := make(Key, 0, len(p.context)+1) + for _, k := range p.context { + keyContext = append(keyContext, k) + } + if len(key) > 0 { // allow type setting for hashes + keyContext = append(keyContext, key) + } + p.types[keyContext.String()] = typ +} + +// addImplicit sets the given Key as having been created implicitly. +func (p *parser) addImplicit(key Key) { + p.implicits[key.String()] = true +} + +// removeImplicit stops tagging the given key as having been implicitly +// created. +func (p *parser) removeImplicit(key Key) { + p.implicits[key.String()] = false +} + +// isImplicit returns true if the key group pointed to by the key was created +// implicitly. +func (p *parser) isImplicit(key Key) bool { + return p.implicits[key.String()] +} + +// current returns the full key name of the current context. +func (p *parser) current() string { + if len(p.currentKey) == 0 { + return p.context.String() + } + if len(p.context) == 0 { + return p.currentKey + } + return fmt.Sprintf("%s.%s", p.context, p.currentKey) +} + +func stripFirstNewline(s string) string { + if len(s) == 0 || s[0] != '\n' { + return s + } + return s[1:len(s)] +} + +func stripEscapedWhitespace(s string) string { + esc := strings.Split(s, "\\\n") + if len(esc) > 1 { + for i := 1; i < len(esc); i++ { + esc[i] = strings.TrimLeftFunc(esc[i], unicode.IsSpace) + } + } + return strings.Join(esc, "") +} + +func (p *parser) replaceEscapes(str string) string { + var replaced []rune + s := []byte(str) + r := 0 + for r < len(s) { + if s[r] != '\\' { + c, size := utf8.DecodeRune(s[r:]) + r += size + replaced = append(replaced, c) + continue + } + r += 1 + if r >= len(s) { + p.bug("Escape sequence at end of string.") + return "" + } + switch s[r] { + default: + p.bug("Expected valid escape code after \\, but got %q.", s[r]) + return "" + case 'b': + replaced = append(replaced, rune(0x0008)) + r += 1 + case 't': + replaced = append(replaced, rune(0x0009)) + r += 1 + case 'n': + replaced = append(replaced, rune(0x000A)) + r += 1 + case 'f': + replaced = append(replaced, rune(0x000C)) + r += 1 + case 'r': + replaced = append(replaced, rune(0x000D)) + r += 1 + case '"': + replaced = append(replaced, rune(0x0022)) + r += 1 + case '\\': + replaced = append(replaced, rune(0x005C)) + r += 1 + case 'u': + // At this point, we know we have a Unicode escape of the form + // `uXXXX` at [r, r+5). (Because the lexer guarantees this + // for us.) + escaped := p.asciiEscapeToUnicode(s[r+1 : r+5]) + replaced = append(replaced, escaped) + r += 5 + case 'U': + // At this point, we know we have a Unicode escape of the form + // `uXXXX` at [r, r+9). (Because the lexer guarantees this + // for us.) + escaped := p.asciiEscapeToUnicode(s[r+1 : r+9]) + replaced = append(replaced, escaped) + r += 9 + } + } + return string(replaced) +} + +func (p *parser) asciiEscapeToUnicode(bs []byte) rune { + s := string(bs) + hex, err := strconv.ParseUint(strings.ToLower(s), 16, 32) + if err != nil { + p.bug("Could not parse '%s' as a hexadecimal number, but the "+ + "lexer claims it's OK: %s", s, err) + } + + // BUG(burntsushi) + // I honestly don't understand how this works. I can't seem + // to find a way to make this fail. I figured this would fail on invalid + // UTF-8 characters like U+DCFF, but it doesn't. + if !utf8.ValidString(string(rune(hex))) { + p.panicf("Escaped character '\\u%s' is not valid UTF-8.", s) + } + return rune(hex) +} + +func isStringType(ty itemType) bool { + return ty == itemString || ty == itemMultilineString || + ty == itemRawString || ty == itemRawMultilineString +} diff --git a/Godeps/_workspace/src/github.com/BurntSushi/toml/session.vim b/Godeps/_workspace/src/github.com/BurntSushi/toml/session.vim new file mode 100644 index 000000000..562164be0 --- /dev/null +++ b/Godeps/_workspace/src/github.com/BurntSushi/toml/session.vim @@ -0,0 +1 @@ +au BufWritePost *.go silent!make tags > /dev/null 2>&1 diff --git a/Godeps/_workspace/src/github.com/BurntSushi/toml/type_check.go b/Godeps/_workspace/src/github.com/BurntSushi/toml/type_check.go new file mode 100644 index 000000000..c73f8afc1 --- /dev/null +++ b/Godeps/_workspace/src/github.com/BurntSushi/toml/type_check.go @@ -0,0 +1,91 @@ +package toml + +// tomlType represents any Go type that corresponds to a TOML type. +// While the first draft of the TOML spec has a simplistic type system that +// probably doesn't need this level of sophistication, we seem to be militating +// toward adding real composite types. +type tomlType interface { + typeString() string +} + +// typeEqual accepts any two types and returns true if they are equal. +func typeEqual(t1, t2 tomlType) bool { + if t1 == nil || t2 == nil { + return false + } + return t1.typeString() == t2.typeString() +} + +func typeIsHash(t tomlType) bool { + return typeEqual(t, tomlHash) || typeEqual(t, tomlArrayHash) +} + +type tomlBaseType string + +func (btype tomlBaseType) typeString() string { + return string(btype) +} + +func (btype tomlBaseType) String() string { + return btype.typeString() +} + +var ( + tomlInteger tomlBaseType = "Integer" + tomlFloat tomlBaseType = "Float" + tomlDatetime tomlBaseType = "Datetime" + tomlString tomlBaseType = "String" + tomlBool tomlBaseType = "Bool" + tomlArray tomlBaseType = "Array" + tomlHash tomlBaseType = "Hash" + tomlArrayHash tomlBaseType = "ArrayHash" +) + +// typeOfPrimitive returns a tomlType of any primitive value in TOML. +// Primitive values are: Integer, Float, Datetime, String and Bool. +// +// Passing a lexer item other than the following will cause a BUG message +// to occur: itemString, itemBool, itemInteger, itemFloat, itemDatetime. +func (p *parser) typeOfPrimitive(lexItem item) tomlType { + switch lexItem.typ { + case itemInteger: + return tomlInteger + case itemFloat: + return tomlFloat + case itemDatetime: + return tomlDatetime + case itemString: + return tomlString + case itemMultilineString: + return tomlString + case itemRawString: + return tomlString + case itemRawMultilineString: + return tomlString + case itemBool: + return tomlBool + } + p.bug("Cannot infer primitive type of lex item '%s'.", lexItem) + panic("unreachable") +} + +// typeOfArray returns a tomlType for an array given a list of types of its +// values. +// +// In the current spec, if an array is homogeneous, then its type is always +// "Array". If the array is not homogeneous, an error is generated. +func (p *parser) typeOfArray(types []tomlType) tomlType { + // Empty arrays are cool. + if len(types) == 0 { + return tomlArray + } + + theType := types[0] + for _, t := range types[1:] { + if !typeEqual(theType, t) { + p.panicf("Array contains values of type '%s' and '%s', but "+ + "arrays must be homogeneous.", theType, t) + } + } + return tomlArray +} diff --git a/Godeps/_workspace/src/github.com/BurntSushi/toml/type_fields.go b/Godeps/_workspace/src/github.com/BurntSushi/toml/type_fields.go new file mode 100644 index 000000000..7592f87a4 --- /dev/null +++ b/Godeps/_workspace/src/github.com/BurntSushi/toml/type_fields.go @@ -0,0 +1,241 @@ +package toml + +// Struct field handling is adapted from code in encoding/json: +// +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the Go distribution. + +import ( + "reflect" + "sort" + "sync" +) + +// A field represents a single field found in a struct. +type field struct { + name string // the name of the field (`toml` tag included) + tag bool // whether field has a `toml` tag + index []int // represents the depth of an anonymous field + typ reflect.Type // the type of the field +} + +// byName sorts field by name, breaking ties with depth, +// then breaking ties with "name came from toml tag", then +// breaking ties with index sequence. +type byName []field + +func (x byName) Len() int { return len(x) } + +func (x byName) Swap(i, j int) { x[i], x[j] = x[j], x[i] } + +func (x byName) Less(i, j int) bool { + if x[i].name != x[j].name { + return x[i].name < x[j].name + } + if len(x[i].index) != len(x[j].index) { + return len(x[i].index) < len(x[j].index) + } + if x[i].tag != x[j].tag { + return x[i].tag + } + return byIndex(x).Less(i, j) +} + +// byIndex sorts field by index sequence. +type byIndex []field + +func (x byIndex) Len() int { return len(x) } + +func (x byIndex) Swap(i, j int) { x[i], x[j] = x[j], x[i] } + +func (x byIndex) Less(i, j int) bool { + for k, xik := range x[i].index { + if k >= len(x[j].index) { + return false + } + if xik != x[j].index[k] { + return xik < x[j].index[k] + } + } + return len(x[i].index) < len(x[j].index) +} + +// typeFields returns a list of fields that TOML should recognize for the given +// type. The algorithm is breadth-first search over the set of structs to +// include - the top struct and then any reachable anonymous structs. +func typeFields(t reflect.Type) []field { + // Anonymous fields to explore at the current level and the next. + current := []field{} + next := []field{{typ: t}} + + // Count of queued names for current level and the next. + count := map[reflect.Type]int{} + nextCount := map[reflect.Type]int{} + + // Types already visited at an earlier level. + visited := map[reflect.Type]bool{} + + // Fields found. + var fields []field + + for len(next) > 0 { + current, next = next, current[:0] + count, nextCount = nextCount, map[reflect.Type]int{} + + for _, f := range current { + if visited[f.typ] { + continue + } + visited[f.typ] = true + + // Scan f.typ for fields to include. + for i := 0; i < f.typ.NumField(); i++ { + sf := f.typ.Field(i) + if sf.PkgPath != "" { // unexported + continue + } + name := sf.Tag.Get("toml") + if name == "-" { + continue + } + index := make([]int, len(f.index)+1) + copy(index, f.index) + index[len(f.index)] = i + + ft := sf.Type + if ft.Name() == "" && ft.Kind() == reflect.Ptr { + // Follow pointer. + ft = ft.Elem() + } + + // Record found field and index sequence. + if name != "" || !sf.Anonymous || ft.Kind() != reflect.Struct { + tagged := name != "" + if name == "" { + name = sf.Name + } + fields = append(fields, field{name, tagged, index, ft}) + if count[f.typ] > 1 { + // If there were multiple instances, add a second, + // so that the annihilation code will see a duplicate. + // It only cares about the distinction between 1 or 2, + // so don't bother generating any more copies. + fields = append(fields, fields[len(fields)-1]) + } + continue + } + + // Record new anonymous struct to explore in next round. + nextCount[ft]++ + if nextCount[ft] == 1 { + f := field{name: ft.Name(), index: index, typ: ft} + next = append(next, f) + } + } + } + } + + sort.Sort(byName(fields)) + + // Delete all fields that are hidden by the Go rules for embedded fields, + // except that fields with TOML tags are promoted. + + // The fields are sorted in primary order of name, secondary order + // of field index length. Loop over names; for each name, delete + // hidden fields by choosing the one dominant field that survives. + out := fields[:0] + for advance, i := 0, 0; i < len(fields); i += advance { + // One iteration per name. + // Find the sequence of fields with the name of this first field. + fi := fields[i] + name := fi.name + for advance = 1; i+advance < len(fields); advance++ { + fj := fields[i+advance] + if fj.name != name { + break + } + } + if advance == 1 { // Only one field with this name + out = append(out, fi) + continue + } + dominant, ok := dominantField(fields[i : i+advance]) + if ok { + out = append(out, dominant) + } + } + + fields = out + sort.Sort(byIndex(fields)) + + return fields +} + +// dominantField looks through the fields, all of which are known to +// have the same name, to find the single field that dominates the +// others using Go's embedding rules, modified by the presence of +// TOML tags. If there are multiple top-level fields, the boolean +// will be false: This condition is an error in Go and we skip all +// the fields. +func dominantField(fields []field) (field, bool) { + // The fields are sorted in increasing index-length order. The winner + // must therefore be one with the shortest index length. Drop all + // longer entries, which is easy: just truncate the slice. + length := len(fields[0].index) + tagged := -1 // Index of first tagged field. + for i, f := range fields { + if len(f.index) > length { + fields = fields[:i] + break + } + if f.tag { + if tagged >= 0 { + // Multiple tagged fields at the same level: conflict. + // Return no field. + return field{}, false + } + tagged = i + } + } + if tagged >= 0 { + return fields[tagged], true + } + // All remaining fields have the same length. If there's more than one, + // we have a conflict (two fields named "X" at the same level) and we + // return no field. + if len(fields) > 1 { + return field{}, false + } + return fields[0], true +} + +var fieldCache struct { + sync.RWMutex + m map[reflect.Type][]field +} + +// cachedTypeFields is like typeFields but uses a cache to avoid repeated work. +func cachedTypeFields(t reflect.Type) []field { + fieldCache.RLock() + f := fieldCache.m[t] + fieldCache.RUnlock() + if f != nil { + return f + } + + // Compute fields without lock. + // Might duplicate effort but won't hold other computations back. + f = typeFields(t) + if f == nil { + f = []field{} + } + + fieldCache.Lock() + if fieldCache.m == nil { + fieldCache.m = map[reflect.Type][]field{} + } + fieldCache.m[t] = f + fieldCache.Unlock() + return f +} diff --git a/Godeps/_workspace/src/github.com/armon/go-metrics/.gitignore b/Godeps/_workspace/src/github.com/armon/go-metrics/.gitignore new file mode 100644 index 000000000..00268614f --- /dev/null +++ b/Godeps/_workspace/src/github.com/armon/go-metrics/.gitignore @@ -0,0 +1,22 @@ +# Compiled Object files, Static and Dynamic libs (Shared Objects) +*.o +*.a +*.so + +# Folders +_obj +_test + +# Architecture specific extensions/prefixes +*.[568vq] +[568vq].out + +*.cgo1.go +*.cgo2.c +_cgo_defun.c +_cgo_gotypes.go +_cgo_export.* + +_testmain.go + +*.exe diff --git a/Godeps/_workspace/src/github.com/armon/go-metrics/LICENSE b/Godeps/_workspace/src/github.com/armon/go-metrics/LICENSE new file mode 100644 index 000000000..106569e54 --- /dev/null +++ b/Godeps/_workspace/src/github.com/armon/go-metrics/LICENSE @@ -0,0 +1,20 @@ +The MIT License (MIT) + +Copyright (c) 2013 Armon Dadgar + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/Godeps/_workspace/src/github.com/armon/go-metrics/README.md b/Godeps/_workspace/src/github.com/armon/go-metrics/README.md new file mode 100644 index 000000000..d9f46e85b --- /dev/null +++ b/Godeps/_workspace/src/github.com/armon/go-metrics/README.md @@ -0,0 +1,68 @@ +go-metrics +========== + +This library provides a `metrics` package which can be used to instrument code, +expose application metrics, and profile runtime performance in a flexible manner. + +Sinks +===== + +The `metrics` package makes use of a `MetricSink` interface to support delivery +to any type of backend. Currently the following sinks are provided: + +* StatsiteSink : Sinks to a statsite instance (TCP) +* StatsdSink: Sinks to a statsd / statsite instance (UDP) +* InmemSink : Provides in-memory aggregation, can be used to export stats +* FanoutSink : Sinks to multiple sinks. Enables writing to multiple statsite instances for example. +* BlackholeSink : Sinks to nowhere + +In addition to the sinks, the `InmemSignal` can be used to catch a signal, +and dump a formatted output of recent metrics. For example, when a process gets +a SIGUSR1, it can dump to stderr recent performance metrics for debugging. + +Examples +======== + +Here is an example of using the package: + + func SlowMethod() { + // Profiling the runtime of a method + defer metrics.MeasureSince([]string{"SlowMethod"}, time.Now()) + } + + // Configure a statsite sink as the global metrics sink + sink, _ := metrics.NewStatsiteSink("statsite:8125") + metrics.NewGlobal(metrics.DefaultConfig("service-name"), sink) + + // Emit a Key/Value pair + metrics.EmitKey([]string{"questions", "meaning of life"}, 42) + + +Here is an example of setting up an signal handler: + + // Setup the inmem sink and signal handler + inm := NewInmemSink(10*time.Second, time.Minute) + sig := DefaultInmemSignal(inm) + metrics.NewGlobal(metrics.DefaultConfig("service-name"), inm) + + // Run some code + inm.SetGauge([]string{"foo"}, 42) + inm.EmitKey([]string{"bar"}, 30) + + inm.IncrCounter([]string{"baz"}, 42) + inm.IncrCounter([]string{"baz"}, 1) + inm.IncrCounter([]string{"baz"}, 80) + + inm.AddSample([]string{"method", "wow"}, 42) + inm.AddSample([]string{"method", "wow"}, 100) + inm.AddSample([]string{"method", "wow"}, 22) + + .... + +When a signal comes in, output like the following will be dumped to stderr: + + [2014-01-28 14:57:33.04 -0800 PST][G] 'foo': 42.000 + [2014-01-28 14:57:33.04 -0800 PST][P] 'bar': 30.000 + [2014-01-28 14:57:33.04 -0800 PST][C] 'baz': Count: 3 Min: 1.000 Mean: 41.000 Max: 80.000 Stddev: 39.509 + [2014-01-28 14:57:33.04 -0800 PST][S] 'method.wow': Count: 3 Min: 22.000 Mean: 54.667 Max: 100.000 Stddev: 40.513 + diff --git a/Godeps/_workspace/src/github.com/armon/go-metrics/const_unix.go b/Godeps/_workspace/src/github.com/armon/go-metrics/const_unix.go new file mode 100644 index 000000000..31098dd57 --- /dev/null +++ b/Godeps/_workspace/src/github.com/armon/go-metrics/const_unix.go @@ -0,0 +1,12 @@ +// +build !windows + +package metrics + +import ( + "syscall" +) + +const ( + // DefaultSignal is used with DefaultInmemSignal + DefaultSignal = syscall.SIGUSR1 +) diff --git a/Godeps/_workspace/src/github.com/armon/go-metrics/const_windows.go b/Godeps/_workspace/src/github.com/armon/go-metrics/const_windows.go new file mode 100644 index 000000000..38136af3e --- /dev/null +++ b/Godeps/_workspace/src/github.com/armon/go-metrics/const_windows.go @@ -0,0 +1,13 @@ +// +build windows + +package metrics + +import ( + "syscall" +) + +const ( + // DefaultSignal is used with DefaultInmemSignal + // Windows has no SIGUSR1, use SIGBREAK + DefaultSignal = syscall.Signal(21) +) diff --git a/Godeps/_workspace/src/github.com/armon/go-metrics/inmem.go b/Godeps/_workspace/src/github.com/armon/go-metrics/inmem.go new file mode 100644 index 000000000..0749229bf --- /dev/null +++ b/Godeps/_workspace/src/github.com/armon/go-metrics/inmem.go @@ -0,0 +1,239 @@ +package metrics + +import ( + "fmt" + "math" + "strings" + "sync" + "time" +) + +// InmemSink provides a MetricSink that does in-memory aggregation +// without sending metrics over a network. It can be embedded within +// an application to provide profiling information. +type InmemSink struct { + // How long is each aggregation interval + interval time.Duration + + // Retain controls how many metrics interval we keep + retain time.Duration + + // maxIntervals is the maximum length of intervals. + // It is retain / interval. + maxIntervals int + + // intervals is a slice of the retained intervals + intervals []*IntervalMetrics + intervalLock sync.RWMutex +} + +// IntervalMetrics stores the aggregated metrics +// for a specific interval +type IntervalMetrics struct { + sync.RWMutex + + // The start time of the interval + Interval time.Time + + // Gauges maps the key to the last set value + Gauges map[string]float32 + + // Points maps the string to the list of emitted values + // from EmitKey + Points map[string][]float32 + + // Counters maps the string key to a sum of the counter + // values + Counters map[string]*AggregateSample + + // Samples maps the key to an AggregateSample, + // which has the rolled up view of a sample + Samples map[string]*AggregateSample +} + +// NewIntervalMetrics creates a new IntervalMetrics for a given interval +func NewIntervalMetrics(intv time.Time) *IntervalMetrics { + return &IntervalMetrics{ + Interval: intv, + Gauges: make(map[string]float32), + Points: make(map[string][]float32), + Counters: make(map[string]*AggregateSample), + Samples: make(map[string]*AggregateSample), + } +} + +// AggregateSample is used to hold aggregate metrics +// about a sample +type AggregateSample struct { + Count int // The count of emitted pairs + Sum float64 // The sum of values + SumSq float64 // The sum of squared values + Min float64 // Minimum value + Max float64 // Maximum value +} + +// Computes a Stddev of the values +func (a *AggregateSample) Stddev() float64 { + num := (float64(a.Count) * a.SumSq) - math.Pow(a.Sum, 2) + div := float64(a.Count * (a.Count - 1)) + if div == 0 { + return 0 + } + return math.Sqrt(num / div) +} + +// Computes a mean of the values +func (a *AggregateSample) Mean() float64 { + if a.Count == 0 { + return 0 + } + return a.Sum / float64(a.Count) +} + +// Ingest is used to update a sample +func (a *AggregateSample) Ingest(v float64) { + a.Count++ + a.Sum += v + a.SumSq += (v * v) + if v < a.Min || a.Count == 1 { + a.Min = v + } + if v > a.Max || a.Count == 1 { + a.Max = v + } +} + +func (a *AggregateSample) String() string { + if a.Count == 0 { + return "Count: 0" + } else if a.Stddev() == 0 { + return fmt.Sprintf("Count: %d Sum: %0.3f", a.Count, a.Sum) + } else { + return fmt.Sprintf("Count: %d Min: %0.3f Mean: %0.3f Max: %0.3f Stddev: %0.3f Sum: %0.3f", + a.Count, a.Min, a.Mean(), a.Max, a.Stddev(), a.Sum) + } +} + +// NewInmemSink is used to construct a new in-memory sink. +// Uses an aggregation interval and maximum retention period. +func NewInmemSink(interval, retain time.Duration) *InmemSink { + i := &InmemSink{ + interval: interval, + retain: retain, + maxIntervals: int(retain / interval), + } + i.intervals = make([]*IntervalMetrics, 0, i.maxIntervals) + return i +} + +func (i *InmemSink) SetGauge(key []string, val float32) { + k := i.flattenKey(key) + intv := i.getInterval() + + intv.Lock() + defer intv.Unlock() + intv.Gauges[k] = val +} + +func (i *InmemSink) EmitKey(key []string, val float32) { + k := i.flattenKey(key) + intv := i.getInterval() + + intv.Lock() + defer intv.Unlock() + vals := intv.Points[k] + intv.Points[k] = append(vals, val) +} + +func (i *InmemSink) IncrCounter(key []string, val float32) { + k := i.flattenKey(key) + intv := i.getInterval() + + intv.Lock() + defer intv.Unlock() + + agg := intv.Counters[k] + if agg == nil { + agg = &AggregateSample{} + intv.Counters[k] = agg + } + agg.Ingest(float64(val)) +} + +func (i *InmemSink) AddSample(key []string, val float32) { + k := i.flattenKey(key) + intv := i.getInterval() + + intv.Lock() + defer intv.Unlock() + + agg := intv.Samples[k] + if agg == nil { + agg = &AggregateSample{} + intv.Samples[k] = agg + } + agg.Ingest(float64(val)) +} + +// Data is used to retrieve all the aggregated metrics +// Intervals may be in use, and a read lock should be acquired +func (i *InmemSink) Data() []*IntervalMetrics { + // Get the current interval, forces creation + i.getInterval() + + i.intervalLock.RLock() + defer i.intervalLock.RUnlock() + + intervals := make([]*IntervalMetrics, len(i.intervals)) + copy(intervals, i.intervals) + return intervals +} + +func (i *InmemSink) getExistingInterval(intv time.Time) *IntervalMetrics { + i.intervalLock.RLock() + defer i.intervalLock.RUnlock() + + n := len(i.intervals) + if n > 0 && i.intervals[n-1].Interval == intv { + return i.intervals[n-1] + } + return nil +} + +func (i *InmemSink) createInterval(intv time.Time) *IntervalMetrics { + i.intervalLock.Lock() + defer i.intervalLock.Unlock() + + // Check for an existing interval + n := len(i.intervals) + if n > 0 && i.intervals[n-1].Interval == intv { + return i.intervals[n-1] + } + + // Add the current interval + current := NewIntervalMetrics(intv) + i.intervals = append(i.intervals, current) + n++ + + // Truncate the intervals if they are too long + if n >= i.maxIntervals { + copy(i.intervals[0:], i.intervals[n-i.maxIntervals:]) + i.intervals = i.intervals[:i.maxIntervals] + } + return current +} + +// getInterval returns the current interval to write to +func (i *InmemSink) getInterval() *IntervalMetrics { + intv := time.Now().Truncate(i.interval) + if m := i.getExistingInterval(intv); m != nil { + return m + } + return i.createInterval(intv) +} + +// Flattens the key for formatting, removes spaces +func (i *InmemSink) flattenKey(parts []string) string { + joined := strings.Join(parts, ".") + return strings.Replace(joined, " ", "_", -1) +} diff --git a/Godeps/_workspace/src/github.com/armon/go-metrics/inmem_signal.go b/Godeps/_workspace/src/github.com/armon/go-metrics/inmem_signal.go new file mode 100644 index 000000000..95d08ee10 --- /dev/null +++ b/Godeps/_workspace/src/github.com/armon/go-metrics/inmem_signal.go @@ -0,0 +1,100 @@ +package metrics + +import ( + "bytes" + "fmt" + "io" + "os" + "os/signal" + "sync" + "syscall" +) + +// InmemSignal is used to listen for a given signal, and when received, +// to dump the current metrics from the InmemSink to an io.Writer +type InmemSignal struct { + signal syscall.Signal + inm *InmemSink + w io.Writer + sigCh chan os.Signal + + stop bool + stopCh chan struct{} + stopLock sync.Mutex +} + +// NewInmemSignal creates a new InmemSignal which listens for a given signal, +// and dumps the current metrics out to a writer +func NewInmemSignal(inmem *InmemSink, sig syscall.Signal, w io.Writer) *InmemSignal { + i := &InmemSignal{ + signal: sig, + inm: inmem, + w: w, + sigCh: make(chan os.Signal, 1), + stopCh: make(chan struct{}), + } + signal.Notify(i.sigCh, sig) + go i.run() + return i +} + +// DefaultInmemSignal returns a new InmemSignal that responds to SIGUSR1 +// and writes output to stderr. Windows uses SIGBREAK +func DefaultInmemSignal(inmem *InmemSink) *InmemSignal { + return NewInmemSignal(inmem, DefaultSignal, os.Stderr) +} + +// Stop is used to stop the InmemSignal from listening +func (i *InmemSignal) Stop() { + i.stopLock.Lock() + defer i.stopLock.Unlock() + + if i.stop { + return + } + i.stop = true + close(i.stopCh) + signal.Stop(i.sigCh) +} + +// run is a long running routine that handles signals +func (i *InmemSignal) run() { + for { + select { + case <-i.sigCh: + i.dumpStats() + case <-i.stopCh: + return + } + } +} + +// dumpStats is used to dump the data to output writer +func (i *InmemSignal) dumpStats() { + buf := bytes.NewBuffer(nil) + + data := i.inm.Data() + // Skip the last period which is still being aggregated + for i := 0; i < len(data)-1; i++ { + intv := data[i] + intv.RLock() + for name, val := range intv.Gauges { + fmt.Fprintf(buf, "[%v][G] '%s': %0.3f\n", intv.Interval, name, val) + } + for name, vals := range intv.Points { + for _, val := range vals { + fmt.Fprintf(buf, "[%v][P] '%s': %0.3f\n", intv.Interval, name, val) + } + } + for name, agg := range intv.Counters { + fmt.Fprintf(buf, "[%v][C] '%s': %s\n", intv.Interval, name, agg) + } + for name, agg := range intv.Samples { + fmt.Fprintf(buf, "[%v][S] '%s': %s\n", intv.Interval, name, agg) + } + intv.RUnlock() + } + + // Write out the bytes + i.w.Write(buf.Bytes()) +} diff --git a/Godeps/_workspace/src/github.com/armon/go-metrics/inmem_signal_test.go b/Godeps/_workspace/src/github.com/armon/go-metrics/inmem_signal_test.go new file mode 100644 index 000000000..9bbca5f25 --- /dev/null +++ b/Godeps/_workspace/src/github.com/armon/go-metrics/inmem_signal_test.go @@ -0,0 +1,46 @@ +package metrics + +import ( + "bytes" + "os" + "strings" + "syscall" + "testing" + "time" +) + +func TestInmemSignal(t *testing.T) { + buf := bytes.NewBuffer(nil) + inm := NewInmemSink(10*time.Millisecond, 50*time.Millisecond) + sig := NewInmemSignal(inm, syscall.SIGUSR1, buf) + defer sig.Stop() + + inm.SetGauge([]string{"foo"}, 42) + inm.EmitKey([]string{"bar"}, 42) + inm.IncrCounter([]string{"baz"}, 42) + inm.AddSample([]string{"wow"}, 42) + + // Wait for period to end + time.Sleep(15 * time.Millisecond) + + // Send signal! + syscall.Kill(os.Getpid(), syscall.SIGUSR1) + + // Wait for flush + time.Sleep(10 * time.Millisecond) + + // Check the output + out := string(buf.Bytes()) + if !strings.Contains(out, "[G] 'foo': 42") { + t.Fatalf("bad: %v", out) + } + if !strings.Contains(out, "[P] 'bar': 42") { + t.Fatalf("bad: %v", out) + } + if !strings.Contains(out, "[C] 'baz': Count: 1 Sum: 42") { + t.Fatalf("bad: %v", out) + } + if !strings.Contains(out, "[S] 'wow': Count: 1 Sum: 42") { + t.Fatalf("bad: %v", out) + } +} diff --git a/Godeps/_workspace/src/github.com/armon/go-metrics/inmem_test.go b/Godeps/_workspace/src/github.com/armon/go-metrics/inmem_test.go new file mode 100644 index 000000000..14ba31b38 --- /dev/null +++ b/Godeps/_workspace/src/github.com/armon/go-metrics/inmem_test.go @@ -0,0 +1,95 @@ +package metrics + +import ( + "math" + "testing" + "time" +) + +func TestInmemSink(t *testing.T) { + inm := NewInmemSink(10*time.Millisecond, 50*time.Millisecond) + + data := inm.Data() + if len(data) != 1 { + t.Fatalf("bad: %v", data) + } + + // Add data points + inm.SetGauge([]string{"foo", "bar"}, 42) + inm.EmitKey([]string{"foo", "bar"}, 42) + inm.IncrCounter([]string{"foo", "bar"}, 20) + inm.IncrCounter([]string{"foo", "bar"}, 22) + inm.AddSample([]string{"foo", "bar"}, 20) + inm.AddSample([]string{"foo", "bar"}, 22) + + data = inm.Data() + if len(data) != 1 { + t.Fatalf("bad: %v", data) + } + + intvM := data[0] + intvM.RLock() + + if time.Now().Sub(intvM.Interval) > 10*time.Millisecond { + t.Fatalf("interval too old") + } + if intvM.Gauges["foo.bar"] != 42 { + t.Fatalf("bad val: %v", intvM.Gauges) + } + if intvM.Points["foo.bar"][0] != 42 { + t.Fatalf("bad val: %v", intvM.Points) + } + + agg := intvM.Counters["foo.bar"] + if agg.Count != 2 { + t.Fatalf("bad val: %v", agg) + } + if agg.Sum != 42 { + t.Fatalf("bad val: %v", agg) + } + if agg.SumSq != 884 { + t.Fatalf("bad val: %v", agg) + } + if agg.Min != 20 { + t.Fatalf("bad val: %v", agg) + } + if agg.Max != 22 { + t.Fatalf("bad val: %v", agg) + } + if agg.Mean() != 21 { + t.Fatalf("bad val: %v", agg) + } + if agg.Stddev() != math.Sqrt(2) { + t.Fatalf("bad val: %v", agg) + } + + if agg = intvM.Samples["foo.bar"]; agg == nil { + t.Fatalf("missing sample") + } + + intvM.RUnlock() + + for i := 1; i < 10; i++ { + time.Sleep(10 * time.Millisecond) + inm.SetGauge([]string{"foo", "bar"}, 42) + data = inm.Data() + if len(data) != min(i+1, 5) { + t.Fatalf("bad: %v", data) + } + } + + // Should not exceed 5 intervals! + time.Sleep(10 * time.Millisecond) + inm.SetGauge([]string{"foo", "bar"}, 42) + data = inm.Data() + if len(data) != 5 { + t.Fatalf("bad: %v", data) + } +} + +func min(a, b int) int { + if a < b { + return a + } + return b +} diff --git a/Godeps/_workspace/src/github.com/armon/go-metrics/metrics.go b/Godeps/_workspace/src/github.com/armon/go-metrics/metrics.go new file mode 100644 index 000000000..b818e4182 --- /dev/null +++ b/Godeps/_workspace/src/github.com/armon/go-metrics/metrics.go @@ -0,0 +1,115 @@ +package metrics + +import ( + "runtime" + "time" +) + +func (m *Metrics) SetGauge(key []string, val float32) { + if m.HostName != "" && m.EnableHostname { + key = insert(0, m.HostName, key) + } + if m.EnableTypePrefix { + key = insert(0, "gauge", key) + } + if m.ServiceName != "" { + key = insert(0, m.ServiceName, key) + } + m.sink.SetGauge(key, val) +} + +func (m *Metrics) EmitKey(key []string, val float32) { + if m.EnableTypePrefix { + key = insert(0, "kv", key) + } + if m.ServiceName != "" { + key = insert(0, m.ServiceName, key) + } + m.sink.EmitKey(key, val) +} + +func (m *Metrics) IncrCounter(key []string, val float32) { + if m.EnableTypePrefix { + key = insert(0, "counter", key) + } + if m.ServiceName != "" { + key = insert(0, m.ServiceName, key) + } + m.sink.IncrCounter(key, val) +} + +func (m *Metrics) AddSample(key []string, val float32) { + if m.EnableTypePrefix { + key = insert(0, "sample", key) + } + if m.ServiceName != "" { + key = insert(0, m.ServiceName, key) + } + m.sink.AddSample(key, val) +} + +func (m *Metrics) MeasureSince(key []string, start time.Time) { + if m.EnableTypePrefix { + key = insert(0, "timer", key) + } + if m.ServiceName != "" { + key = insert(0, m.ServiceName, key) + } + now := time.Now() + elapsed := now.Sub(start) + msec := float32(elapsed.Nanoseconds()) / float32(m.TimerGranularity) + m.sink.AddSample(key, msec) +} + +// Periodically collects runtime stats to publish +func (m *Metrics) collectStats() { + for { + time.Sleep(m.ProfileInterval) + m.emitRuntimeStats() + } +} + +// Emits various runtime statsitics +func (m *Metrics) emitRuntimeStats() { + // Export number of Goroutines + numRoutines := runtime.NumGoroutine() + m.SetGauge([]string{"runtime", "num_goroutines"}, float32(numRoutines)) + + // Export memory stats + var stats runtime.MemStats + runtime.ReadMemStats(&stats) + m.SetGauge([]string{"runtime", "alloc_bytes"}, float32(stats.Alloc)) + m.SetGauge([]string{"runtime", "sys_bytes"}, float32(stats.Sys)) + m.SetGauge([]string{"runtime", "malloc_count"}, float32(stats.Mallocs)) + m.SetGauge([]string{"runtime", "free_count"}, float32(stats.Frees)) + m.SetGauge([]string{"runtime", "heap_objects"}, float32(stats.HeapObjects)) + m.SetGauge([]string{"runtime", "total_gc_pause_ns"}, float32(stats.PauseTotalNs)) + m.SetGauge([]string{"runtime", "total_gc_runs"}, float32(stats.NumGC)) + + // Export info about the last few GC runs + num := stats.NumGC + + // Handle wrap around + if num < m.lastNumGC { + m.lastNumGC = 0 + } + + // Ensure we don't scan more than 256 + if num-m.lastNumGC >= 256 { + m.lastNumGC = num - 255 + } + + for i := m.lastNumGC; i < num; i++ { + pause := stats.PauseNs[i%256] + m.AddSample([]string{"runtime", "gc_pause_ns"}, float32(pause)) + } + m.lastNumGC = num +} + +// Inserts a string value at an index into the slice +func insert(i int, v string, s []string) []string { + s = append(s, "") + copy(s[i+1:], s[i:]) + s[i] = v + return s +} diff --git a/Godeps/_workspace/src/github.com/armon/go-metrics/metrics_test.go b/Godeps/_workspace/src/github.com/armon/go-metrics/metrics_test.go new file mode 100644 index 000000000..9d7558ea9 --- /dev/null +++ b/Godeps/_workspace/src/github.com/armon/go-metrics/metrics_test.go @@ -0,0 +1,262 @@ +package metrics + +import ( + "reflect" + "runtime" + "testing" + "time" +) + +func mockMetric() (*MockSink, *Metrics) { + m := &MockSink{} + met := &Metrics{sink: m} + return m, met +} + +func TestMetrics_SetGauge(t *testing.T) { + m, met := mockMetric() + met.SetGauge([]string{"key"}, float32(1)) + if m.keys[0][0] != "key" { + t.Fatalf("") + } + if m.vals[0] != 1 { + t.Fatalf("") + } + + m, met = mockMetric() + met.HostName = "test" + met.EnableHostname = true + met.SetGauge([]string{"key"}, float32(1)) + if m.keys[0][0] != "test" || m.keys[0][1] != "key" { + t.Fatalf("") + } + if m.vals[0] != 1 { + t.Fatalf("") + } + + m, met = mockMetric() + met.EnableTypePrefix = true + met.SetGauge([]string{"key"}, float32(1)) + if m.keys[0][0] != "gauge" || m.keys[0][1] != "key" { + t.Fatalf("") + } + if m.vals[0] != 1 { + t.Fatalf("") + } + + m, met = mockMetric() + met.ServiceName = "service" + met.SetGauge([]string{"key"}, float32(1)) + if m.keys[0][0] != "service" || m.keys[0][1] != "key" { + t.Fatalf("") + } + if m.vals[0] != 1 { + t.Fatalf("") + } +} + +func TestMetrics_EmitKey(t *testing.T) { + m, met := mockMetric() + met.EmitKey([]string{"key"}, float32(1)) + if m.keys[0][0] != "key" { + t.Fatalf("") + } + if m.vals[0] != 1 { + t.Fatalf("") + } + + m, met = mockMetric() + met.EnableTypePrefix = true + met.EmitKey([]string{"key"}, float32(1)) + if m.keys[0][0] != "kv" || m.keys[0][1] != "key" { + t.Fatalf("") + } + if m.vals[0] != 1 { + t.Fatalf("") + } + + m, met = mockMetric() + met.ServiceName = "service" + met.EmitKey([]string{"key"}, float32(1)) + if m.keys[0][0] != "service" || m.keys[0][1] != "key" { + t.Fatalf("") + } + if m.vals[0] != 1 { + t.Fatalf("") + } +} + +func TestMetrics_IncrCounter(t *testing.T) { + m, met := mockMetric() + met.IncrCounter([]string{"key"}, float32(1)) + if m.keys[0][0] != "key" { + t.Fatalf("") + } + if m.vals[0] != 1 { + t.Fatalf("") + } + + m, met = mockMetric() + met.EnableTypePrefix = true + met.IncrCounter([]string{"key"}, float32(1)) + if m.keys[0][0] != "counter" || m.keys[0][1] != "key" { + t.Fatalf("") + } + if m.vals[0] != 1 { + t.Fatalf("") + } + + m, met = mockMetric() + met.ServiceName = "service" + met.IncrCounter([]string{"key"}, float32(1)) + if m.keys[0][0] != "service" || m.keys[0][1] != "key" { + t.Fatalf("") + } + if m.vals[0] != 1 { + t.Fatalf("") + } +} + +func TestMetrics_AddSample(t *testing.T) { + m, met := mockMetric() + met.AddSample([]string{"key"}, float32(1)) + if m.keys[0][0] != "key" { + t.Fatalf("") + } + if m.vals[0] != 1 { + t.Fatalf("") + } + + m, met = mockMetric() + met.EnableTypePrefix = true + met.AddSample([]string{"key"}, float32(1)) + if m.keys[0][0] != "sample" || m.keys[0][1] != "key" { + t.Fatalf("") + } + if m.vals[0] != 1 { + t.Fatalf("") + } + + m, met = mockMetric() + met.ServiceName = "service" + met.AddSample([]string{"key"}, float32(1)) + if m.keys[0][0] != "service" || m.keys[0][1] != "key" { + t.Fatalf("") + } + if m.vals[0] != 1 { + t.Fatalf("") + } +} + +func TestMetrics_MeasureSince(t *testing.T) { + m, met := mockMetric() + met.TimerGranularity = time.Millisecond + n := time.Now() + met.MeasureSince([]string{"key"}, n) + if m.keys[0][0] != "key" { + t.Fatalf("") + } + if m.vals[0] > 0.1 { + t.Fatalf("") + } + + m, met = mockMetric() + met.TimerGranularity = time.Millisecond + met.EnableTypePrefix = true + met.MeasureSince([]string{"key"}, n) + if m.keys[0][0] != "timer" || m.keys[0][1] != "key" { + t.Fatalf("") + } + if m.vals[0] > 0.1 { + t.Fatalf("") + } + + m, met = mockMetric() + met.TimerGranularity = time.Millisecond + met.ServiceName = "service" + met.MeasureSince([]string{"key"}, n) + if m.keys[0][0] != "service" || m.keys[0][1] != "key" { + t.Fatalf("") + } + if m.vals[0] > 0.1 { + t.Fatalf("") + } +} + +func TestMetrics_EmitRuntimeStats(t *testing.T) { + runtime.GC() + m, met := mockMetric() + met.emitRuntimeStats() + + if m.keys[0][0] != "runtime" || m.keys[0][1] != "num_goroutines" { + t.Fatalf("bad key %v", m.keys) + } + if m.vals[0] <= 1 { + t.Fatalf("bad val: %v", m.vals) + } + + if m.keys[1][0] != "runtime" || m.keys[1][1] != "alloc_bytes" { + t.Fatalf("bad key %v", m.keys) + } + if m.vals[1] <= 100000 { + t.Fatalf("bad val: %v", m.vals) + } + + if m.keys[2][0] != "runtime" || m.keys[2][1] != "sys_bytes" { + t.Fatalf("bad key %v", m.keys) + } + if m.vals[2] <= 100000 { + t.Fatalf("bad val: %v", m.vals) + } + + if m.keys[3][0] != "runtime" || m.keys[3][1] != "malloc_count" { + t.Fatalf("bad key %v", m.keys) + } + if m.vals[3] <= 100 { + t.Fatalf("bad val: %v", m.vals) + } + + if m.keys[4][0] != "runtime" || m.keys[4][1] != "free_count" { + t.Fatalf("bad key %v", m.keys) + } + if m.vals[4] <= 100 { + t.Fatalf("bad val: %v", m.vals) + } + + if m.keys[5][0] != "runtime" || m.keys[5][1] != "heap_objects" { + t.Fatalf("bad key %v", m.keys) + } + if m.vals[5] <= 200 { + t.Fatalf("bad val: %v", m.vals) + } + + if m.keys[6][0] != "runtime" || m.keys[6][1] != "total_gc_pause_ns" { + t.Fatalf("bad key %v", m.keys) + } + if m.vals[6] <= 100000 { + t.Fatalf("bad val: %v", m.vals) + } + + if m.keys[7][0] != "runtime" || m.keys[7][1] != "total_gc_runs" { + t.Fatalf("bad key %v", m.keys) + } + if m.vals[7] <= 1 { + t.Fatalf("bad val: %v", m.vals) + } + + if m.keys[8][0] != "runtime" || m.keys[8][1] != "gc_pause_ns" { + t.Fatalf("bad key %v", m.keys) + } + if m.vals[8] <= 1000 { + t.Fatalf("bad val: %v", m.vals) + } +} + +func TestInsert(t *testing.T) { + k := []string{"hi", "bob"} + exp := []string{"hi", "there", "bob"} + out := insert(1, "there", k) + if !reflect.DeepEqual(exp, out) { + t.Fatalf("bad insert %v %v", exp, out) + } +} diff --git a/Godeps/_workspace/src/github.com/armon/go-metrics/sink.go b/Godeps/_workspace/src/github.com/armon/go-metrics/sink.go new file mode 100644 index 000000000..0c240c2c4 --- /dev/null +++ b/Godeps/_workspace/src/github.com/armon/go-metrics/sink.go @@ -0,0 +1,52 @@ +package metrics + +// The MetricSink interface is used to transmit metrics information +// to an external system +type MetricSink interface { + // A Gauge should retain the last value it is set to + SetGauge(key []string, val float32) + + // Should emit a Key/Value pair for each call + EmitKey(key []string, val float32) + + // Counters should accumulate values + IncrCounter(key []string, val float32) + + // Samples are for timing information, where quantiles are used + AddSample(key []string, val float32) +} + +// BlackholeSink is used to just blackhole messages +type BlackholeSink struct{} + +func (*BlackholeSink) SetGauge(key []string, val float32) {} +func (*BlackholeSink) EmitKey(key []string, val float32) {} +func (*BlackholeSink) IncrCounter(key []string, val float32) {} +func (*BlackholeSink) AddSample(key []string, val float32) {} + +// FanoutSink is used to sink to fanout values to multiple sinks +type FanoutSink []MetricSink + +func (fh FanoutSink) SetGauge(key []string, val float32) { + for _, s := range fh { + s.SetGauge(key, val) + } +} + +func (fh FanoutSink) EmitKey(key []string, val float32) { + for _, s := range fh { + s.EmitKey(key, val) + } +} + +func (fh FanoutSink) IncrCounter(key []string, val float32) { + for _, s := range fh { + s.IncrCounter(key, val) + } +} + +func (fh FanoutSink) AddSample(key []string, val float32) { + for _, s := range fh { + s.AddSample(key, val) + } +} diff --git a/Godeps/_workspace/src/github.com/armon/go-metrics/sink_test.go b/Godeps/_workspace/src/github.com/armon/go-metrics/sink_test.go new file mode 100644 index 000000000..15c5d771a --- /dev/null +++ b/Godeps/_workspace/src/github.com/armon/go-metrics/sink_test.go @@ -0,0 +1,120 @@ +package metrics + +import ( + "reflect" + "testing" +) + +type MockSink struct { + keys [][]string + vals []float32 +} + +func (m *MockSink) SetGauge(key []string, val float32) { + m.keys = append(m.keys, key) + m.vals = append(m.vals, val) +} +func (m *MockSink) EmitKey(key []string, val float32) { + m.keys = append(m.keys, key) + m.vals = append(m.vals, val) +} +func (m *MockSink) IncrCounter(key []string, val float32) { + m.keys = append(m.keys, key) + m.vals = append(m.vals, val) +} +func (m *MockSink) AddSample(key []string, val float32) { + m.keys = append(m.keys, key) + m.vals = append(m.vals, val) +} + +func TestFanoutSink_Gauge(t *testing.T) { + m1 := &MockSink{} + m2 := &MockSink{} + fh := &FanoutSink{m1, m2} + + k := []string{"test"} + v := float32(42.0) + fh.SetGauge(k, v) + + if !reflect.DeepEqual(m1.keys[0], k) { + t.Fatalf("key not equal") + } + if !reflect.DeepEqual(m2.keys[0], k) { + t.Fatalf("key not equal") + } + if !reflect.DeepEqual(m1.vals[0], v) { + t.Fatalf("val not equal") + } + if !reflect.DeepEqual(m2.vals[0], v) { + t.Fatalf("val not equal") + } +} + +func TestFanoutSink_Key(t *testing.T) { + m1 := &MockSink{} + m2 := &MockSink{} + fh := &FanoutSink{m1, m2} + + k := []string{"test"} + v := float32(42.0) + fh.EmitKey(k, v) + + if !reflect.DeepEqual(m1.keys[0], k) { + t.Fatalf("key not equal") + } + if !reflect.DeepEqual(m2.keys[0], k) { + t.Fatalf("key not equal") + } + if !reflect.DeepEqual(m1.vals[0], v) { + t.Fatalf("val not equal") + } + if !reflect.DeepEqual(m2.vals[0], v) { + t.Fatalf("val not equal") + } +} + +func TestFanoutSink_Counter(t *testing.T) { + m1 := &MockSink{} + m2 := &MockSink{} + fh := &FanoutSink{m1, m2} + + k := []string{"test"} + v := float32(42.0) + fh.IncrCounter(k, v) + + if !reflect.DeepEqual(m1.keys[0], k) { + t.Fatalf("key not equal") + } + if !reflect.DeepEqual(m2.keys[0], k) { + t.Fatalf("key not equal") + } + if !reflect.DeepEqual(m1.vals[0], v) { + t.Fatalf("val not equal") + } + if !reflect.DeepEqual(m2.vals[0], v) { + t.Fatalf("val not equal") + } +} + +func TestFanoutSink_Sample(t *testing.T) { + m1 := &MockSink{} + m2 := &MockSink{} + fh := &FanoutSink{m1, m2} + + k := []string{"test"} + v := float32(42.0) + fh.AddSample(k, v) + + if !reflect.DeepEqual(m1.keys[0], k) { + t.Fatalf("key not equal") + } + if !reflect.DeepEqual(m2.keys[0], k) { + t.Fatalf("key not equal") + } + if !reflect.DeepEqual(m1.vals[0], v) { + t.Fatalf("val not equal") + } + if !reflect.DeepEqual(m2.vals[0], v) { + t.Fatalf("val not equal") + } +} diff --git a/Godeps/_workspace/src/github.com/armon/go-metrics/start.go b/Godeps/_workspace/src/github.com/armon/go-metrics/start.go new file mode 100644 index 000000000..44113f100 --- /dev/null +++ b/Godeps/_workspace/src/github.com/armon/go-metrics/start.go @@ -0,0 +1,95 @@ +package metrics + +import ( + "os" + "time" +) + +// Config is used to configure metrics settings +type Config struct { + ServiceName string // Prefixed with keys to seperate services + HostName string // Hostname to use. If not provided and EnableHostname, it will be os.Hostname + EnableHostname bool // Enable prefixing gauge values with hostname + EnableRuntimeMetrics bool // Enables profiling of runtime metrics (GC, Goroutines, Memory) + EnableTypePrefix bool // Prefixes key with a type ("counter", "gauge", "timer") + TimerGranularity time.Duration // Granularity of timers. + ProfileInterval time.Duration // Interval to profile runtime metrics +} + +// Metrics represents an instance of a metrics sink that can +// be used to emit +type Metrics struct { + Config + lastNumGC uint32 + sink MetricSink +} + +// Shared global metrics instance +var globalMetrics *Metrics + +func init() { + // Initialize to a blackhole sink to avoid errors + globalMetrics = &Metrics{sink: &BlackholeSink{}} +} + +// DefaultConfig provides a sane default configuration +func DefaultConfig(serviceName string) *Config { + c := &Config{ + ServiceName: serviceName, // Use client provided service + HostName: "", + EnableHostname: true, // Enable hostname prefix + EnableRuntimeMetrics: true, // Enable runtime profiling + EnableTypePrefix: false, // Disable type prefix + TimerGranularity: time.Millisecond, // Timers are in milliseconds + ProfileInterval: time.Second, // Poll runtime every second + } + + // Try to get the hostname + name, _ := os.Hostname() + c.HostName = name + return c +} + +// New is used to create a new instance of Metrics +func New(conf *Config, sink MetricSink) (*Metrics, error) { + met := &Metrics{} + met.Config = *conf + met.sink = sink + + // Start the runtime collector + if conf.EnableRuntimeMetrics { + go met.collectStats() + } + return met, nil +} + +// NewGlobal is the same as New, but it assigns the metrics object to be +// used globally as well as returning it. +func NewGlobal(conf *Config, sink MetricSink) (*Metrics, error) { + metrics, err := New(conf, sink) + if err == nil { + globalMetrics = metrics + } + return metrics, err +} + +// Proxy all the methods to the globalMetrics instance +func SetGauge(key []string, val float32) { + globalMetrics.SetGauge(key, val) +} + +func EmitKey(key []string, val float32) { + globalMetrics.EmitKey(key, val) +} + +func IncrCounter(key []string, val float32) { + globalMetrics.IncrCounter(key, val) +} + +func AddSample(key []string, val float32) { + globalMetrics.AddSample(key, val) +} + +func MeasureSince(key []string, start time.Time) { + globalMetrics.MeasureSince(key, start) +} diff --git a/Godeps/_workspace/src/github.com/armon/go-metrics/start_test.go b/Godeps/_workspace/src/github.com/armon/go-metrics/start_test.go new file mode 100644 index 000000000..8b3210c15 --- /dev/null +++ b/Godeps/_workspace/src/github.com/armon/go-metrics/start_test.go @@ -0,0 +1,110 @@ +package metrics + +import ( + "reflect" + "testing" + "time" +) + +func TestDefaultConfig(t *testing.T) { + conf := DefaultConfig("service") + if conf.ServiceName != "service" { + t.Fatalf("Bad name") + } + if conf.HostName == "" { + t.Fatalf("missing hostname") + } + if !conf.EnableHostname || !conf.EnableRuntimeMetrics { + t.Fatalf("expect true") + } + if conf.EnableTypePrefix { + t.Fatalf("expect false") + } + if conf.TimerGranularity != time.Millisecond { + t.Fatalf("bad granularity") + } + if conf.ProfileInterval != time.Second { + t.Fatalf("bad interval") + } +} + +func Test_GlobalMetrics_SetGauge(t *testing.T) { + m := &MockSink{} + globalMetrics = &Metrics{sink: m} + + k := []string{"test"} + v := float32(42.0) + SetGauge(k, v) + + if !reflect.DeepEqual(m.keys[0], k) { + t.Fatalf("key not equal") + } + if !reflect.DeepEqual(m.vals[0], v) { + t.Fatalf("val not equal") + } +} + +func Test_GlobalMetrics_EmitKey(t *testing.T) { + m := &MockSink{} + globalMetrics = &Metrics{sink: m} + + k := []string{"test"} + v := float32(42.0) + EmitKey(k, v) + + if !reflect.DeepEqual(m.keys[0], k) { + t.Fatalf("key not equal") + } + if !reflect.DeepEqual(m.vals[0], v) { + t.Fatalf("val not equal") + } +} + +func Test_GlobalMetrics_IncrCounter(t *testing.T) { + m := &MockSink{} + globalMetrics = &Metrics{sink: m} + + k := []string{"test"} + v := float32(42.0) + IncrCounter(k, v) + + if !reflect.DeepEqual(m.keys[0], k) { + t.Fatalf("key not equal") + } + if !reflect.DeepEqual(m.vals[0], v) { + t.Fatalf("val not equal") + } +} + +func Test_GlobalMetrics_AddSample(t *testing.T) { + m := &MockSink{} + globalMetrics = &Metrics{sink: m} + + k := []string{"test"} + v := float32(42.0) + AddSample(k, v) + + if !reflect.DeepEqual(m.keys[0], k) { + t.Fatalf("key not equal") + } + if !reflect.DeepEqual(m.vals[0], v) { + t.Fatalf("val not equal") + } +} + +func Test_GlobalMetrics_MeasureSince(t *testing.T) { + m := &MockSink{} + globalMetrics = &Metrics{sink: m} + globalMetrics.TimerGranularity = time.Millisecond + + k := []string{"test"} + now := time.Now() + MeasureSince(k, now) + + if !reflect.DeepEqual(m.keys[0], k) { + t.Fatalf("key not equal") + } + if m.vals[0] > 0.1 { + t.Fatalf("val too large %v", m.vals[0]) + } +} diff --git a/Godeps/_workspace/src/github.com/armon/go-metrics/statsd.go b/Godeps/_workspace/src/github.com/armon/go-metrics/statsd.go new file mode 100644 index 000000000..65a5021a0 --- /dev/null +++ b/Godeps/_workspace/src/github.com/armon/go-metrics/statsd.go @@ -0,0 +1,154 @@ +package metrics + +import ( + "bytes" + "fmt" + "log" + "net" + "strings" + "time" +) + +const ( + // statsdMaxLen is the maximum size of a packet + // to send to statsd + statsdMaxLen = 1400 +) + +// StatsdSink provides a MetricSink that can be used +// with a statsite or statsd metrics server. It uses +// only UDP packets, while StatsiteSink uses TCP. +type StatsdSink struct { + addr string + metricQueue chan string +} + +// NewStatsdSink is used to create a new StatsdSink +func NewStatsdSink(addr string) (*StatsdSink, error) { + s := &StatsdSink{ + addr: addr, + metricQueue: make(chan string, 4096), + } + go s.flushMetrics() + return s, nil +} + +// Close is used to stop flushing to statsd +func (s *StatsdSink) Shutdown() { + close(s.metricQueue) +} + +func (s *StatsdSink) SetGauge(key []string, val float32) { + flatKey := s.flattenKey(key) + s.pushMetric(fmt.Sprintf("%s:%f|g\n", flatKey, val)) +} + +func (s *StatsdSink) EmitKey(key []string, val float32) { + flatKey := s.flattenKey(key) + s.pushMetric(fmt.Sprintf("%s:%f|kv\n", flatKey, val)) +} + +func (s *StatsdSink) IncrCounter(key []string, val float32) { + flatKey := s.flattenKey(key) + s.pushMetric(fmt.Sprintf("%s:%f|c\n", flatKey, val)) +} + +func (s *StatsdSink) AddSample(key []string, val float32) { + flatKey := s.flattenKey(key) + s.pushMetric(fmt.Sprintf("%s:%f|ms\n", flatKey, val)) +} + +// Flattens the key for formatting, removes spaces +func (s *StatsdSink) flattenKey(parts []string) string { + joined := strings.Join(parts, ".") + return strings.Map(func(r rune) rune { + switch r { + case ':': + fallthrough + case ' ': + return '_' + default: + return r + } + }, joined) +} + +// Does a non-blocking push to the metrics queue +func (s *StatsdSink) pushMetric(m string) { + select { + case s.metricQueue <- m: + default: + } +} + +// Flushes metrics +func (s *StatsdSink) flushMetrics() { + var sock net.Conn + var err error + var wait <-chan time.Time + ticker := time.NewTicker(flushInterval) + defer ticker.Stop() + +CONNECT: + // Create a buffer + buf := bytes.NewBuffer(nil) + + // Attempt to connect + sock, err = net.Dial("udp", s.addr) + if err != nil { + log.Printf("[ERR] Error connecting to statsd! Err: %s", err) + goto WAIT + } + + for { + select { + case metric, ok := <-s.metricQueue: + // Get a metric from the queue + if !ok { + goto QUIT + } + + // Check if this would overflow the packet size + if len(metric)+buf.Len() > statsdMaxLen { + _, err := sock.Write(buf.Bytes()) + buf.Reset() + if err != nil { + log.Printf("[ERR] Error writing to statsd! Err: %s", err) + goto WAIT + } + } + + // Append to the buffer + buf.WriteString(metric) + + case <-ticker.C: + if buf.Len() == 0 { + continue + } + + _, err := sock.Write(buf.Bytes()) + buf.Reset() + if err != nil { + log.Printf("[ERR] Error flushing to statsd! Err: %s", err) + goto WAIT + } + } + } + +WAIT: + // Wait for a while + wait = time.After(time.Duration(5) * time.Second) + for { + select { + // Dequeue the messages to avoid backlog + case _, ok := <-s.metricQueue: + if !ok { + goto QUIT + } + case <-wait: + goto CONNECT + } + } +QUIT: + s.metricQueue = nil +} diff --git a/Godeps/_workspace/src/github.com/armon/go-metrics/statsd_test.go b/Godeps/_workspace/src/github.com/armon/go-metrics/statsd_test.go new file mode 100644 index 000000000..622eb5d3a --- /dev/null +++ b/Godeps/_workspace/src/github.com/armon/go-metrics/statsd_test.go @@ -0,0 +1,105 @@ +package metrics + +import ( + "bufio" + "bytes" + "net" + "testing" + "time" +) + +func TestStatsd_Flatten(t *testing.T) { + s := &StatsdSink{} + flat := s.flattenKey([]string{"a", "b", "c", "d"}) + if flat != "a.b.c.d" { + t.Fatalf("Bad flat") + } +} + +func TestStatsd_PushFullQueue(t *testing.T) { + q := make(chan string, 1) + q <- "full" + + s := &StatsdSink{metricQueue: q} + s.pushMetric("omit") + + out := <-q + if out != "full" { + t.Fatalf("bad val %v", out) + } + + select { + case v := <-q: + t.Fatalf("bad val %v", v) + default: + } +} + +func TestStatsd_Conn(t *testing.T) { + addr := "127.0.0.1:7524" + done := make(chan bool) + go func() { + list, err := net.ListenUDP("udp", &net.UDPAddr{IP: net.ParseIP("127.0.0.1"), Port: 7524}) + if err != nil { + panic(err) + } + defer list.Close() + buf := make([]byte, 1500) + n, err := list.Read(buf) + if err != nil { + panic(err) + } + buf = buf[:n] + reader := bufio.NewReader(bytes.NewReader(buf)) + + line, err := reader.ReadString('\n') + if err != nil { + t.Fatalf("unexpected err %s", err) + } + if line != "gauge.val:1.000000|g\n" { + t.Fatalf("bad line %s", line) + } + + line, err = reader.ReadString('\n') + if err != nil { + t.Fatalf("unexpected err %s", err) + } + if line != "key.other:2.000000|kv\n" { + t.Fatalf("bad line %s", line) + } + + line, err = reader.ReadString('\n') + if err != nil { + t.Fatalf("unexpected err %s", err) + } + if line != "counter.me:3.000000|c\n" { + t.Fatalf("bad line %s", line) + } + + line, err = reader.ReadString('\n') + if err != nil { + t.Fatalf("unexpected err %s", err) + } + if line != "sample.slow_thingy:4.000000|ms\n" { + t.Fatalf("bad line %s", line) + } + + done <- true + }() + s, err := NewStatsdSink(addr) + if err != nil { + t.Fatalf("bad error") + } + + s.SetGauge([]string{"gauge", "val"}, float32(1)) + s.EmitKey([]string{"key", "other"}, float32(2)) + s.IncrCounter([]string{"counter", "me"}, float32(3)) + s.AddSample([]string{"sample", "slow thingy"}, float32(4)) + + select { + case <-done: + s.Shutdown() + case <-time.After(3 * time.Second): + t.Fatalf("timeout") + } +} diff --git a/Godeps/_workspace/src/github.com/armon/go-metrics/statsite.go b/Godeps/_workspace/src/github.com/armon/go-metrics/statsite.go new file mode 100644 index 000000000..68730139a --- /dev/null +++ b/Godeps/_workspace/src/github.com/armon/go-metrics/statsite.go @@ -0,0 +1,142 @@ +package metrics + +import ( + "bufio" + "fmt" + "log" + "net" + "strings" + "time" +) + +const ( + // We force flush the statsite metrics after this period of + // inactivity. Prevents stats from getting stuck in a buffer + // forever. + flushInterval = 100 * time.Millisecond +) + +// StatsiteSink provides a MetricSink that can be used with a +// statsite metrics server +type StatsiteSink struct { + addr string + metricQueue chan string +} + +// NewStatsiteSink is used to create a new StatsiteSink +func NewStatsiteSink(addr string) (*StatsiteSink, error) { + s := &StatsiteSink{ + addr: addr, + metricQueue: make(chan string, 4096), + } + go s.flushMetrics() + return s, nil +} + +// Close is used to stop flushing to statsite +func (s *StatsiteSink) Shutdown() { + close(s.metricQueue) +} + +func (s *StatsiteSink) SetGauge(key []string, val float32) { + flatKey := s.flattenKey(key) + s.pushMetric(fmt.Sprintf("%s:%f|g\n", flatKey, val)) +} + +func (s *StatsiteSink) EmitKey(key []string, val float32) { + flatKey := s.flattenKey(key) + s.pushMetric(fmt.Sprintf("%s:%f|kv\n", flatKey, val)) +} + +func (s *StatsiteSink) IncrCounter(key []string, val float32) { + flatKey := s.flattenKey(key) + s.pushMetric(fmt.Sprintf("%s:%f|c\n", flatKey, val)) +} + +func (s *StatsiteSink) AddSample(key []string, val float32) { + flatKey := s.flattenKey(key) + s.pushMetric(fmt.Sprintf("%s:%f|ms\n", flatKey, val)) +} + +// Flattens the key for formatting, removes spaces +func (s *StatsiteSink) flattenKey(parts []string) string { + joined := strings.Join(parts, ".") + return strings.Map(func(r rune) rune { + switch r { + case ':': + fallthrough + case ' ': + return '_' + default: + return r + } + }, joined) +} + +// Does a non-blocking push to the metrics queue +func (s *StatsiteSink) pushMetric(m string) { + select { + case s.metricQueue <- m: + default: + } +} + +// Flushes metrics +func (s *StatsiteSink) flushMetrics() { + var sock net.Conn + var err error + var wait <-chan time.Time + var buffered *bufio.Writer + ticker := time.NewTicker(flushInterval) + defer ticker.Stop() + +CONNECT: + // Attempt to connect + sock, err = net.Dial("tcp", s.addr) + if err != nil { + log.Printf("[ERR] Error connecting to statsite! Err: %s", err) + goto WAIT + } + + // Create a buffered writer + buffered = bufio.NewWriter(sock) + + for { + select { + case metric, ok := <-s.metricQueue: + // Get a metric from the queue + if !ok { + goto QUIT + } + + // Try to send to statsite + _, err := buffered.Write([]byte(metric)) + if err != nil { + log.Printf("[ERR] Error writing to statsite! Err: %s", err) + goto WAIT + } + case <-ticker.C: + if err := buffered.Flush(); err != nil { + log.Printf("[ERR] Error flushing to statsite! Err: %s", err) + goto WAIT + } + } + } + +WAIT: + // Wait for a while + wait = time.After(time.Duration(5) * time.Second) + for { + select { + // Dequeue the messages to avoid backlog + case _, ok := <-s.metricQueue: + if !ok { + goto QUIT + } + case <-wait: + goto CONNECT + } + } +QUIT: + s.metricQueue = nil +} diff --git a/Godeps/_workspace/src/github.com/armon/go-metrics/statsite_test.go b/Godeps/_workspace/src/github.com/armon/go-metrics/statsite_test.go new file mode 100644 index 000000000..d9c744f41 --- /dev/null +++ b/Godeps/_workspace/src/github.com/armon/go-metrics/statsite_test.go @@ -0,0 +1,101 @@ +package metrics + +import ( + "bufio" + "net" + "testing" + "time" +) + +func acceptConn(addr string) net.Conn { + ln, _ := net.Listen("tcp", addr) + conn, _ := ln.Accept() + return conn +} + +func TestStatsite_Flatten(t *testing.T) { + s := &StatsiteSink{} + flat := s.flattenKey([]string{"a", "b", "c", "d"}) + if flat != "a.b.c.d" { + t.Fatalf("Bad flat") + } +} + +func TestStatsite_PushFullQueue(t *testing.T) { + q := make(chan string, 1) + q <- "full" + + s := &StatsiteSink{metricQueue: q} + s.pushMetric("omit") + + out := <-q + if out != "full" { + t.Fatalf("bad val %v", out) + } + + select { + case v := <-q: + t.Fatalf("bad val %v", v) + default: + } +} + +func TestStatsite_Conn(t *testing.T) { + addr := "localhost:7523" + done := make(chan bool) + go func() { + conn := acceptConn(addr) + reader := bufio.NewReader(conn) + + line, err := reader.ReadString('\n') + if err != nil { + t.Fatalf("unexpected err %s", err) + } + if line != "gauge.val:1.000000|g\n" { + t.Fatalf("bad line %s", line) + } + + line, err = reader.ReadString('\n') + if err != nil { + t.Fatalf("unexpected err %s", err) + } + if line != "key.other:2.000000|kv\n" { + t.Fatalf("bad line %s", line) + } + + line, err = reader.ReadString('\n') + if err != nil { + t.Fatalf("unexpected err %s", err) + } + if line != "counter.me:3.000000|c\n" { + t.Fatalf("bad line %s", line) + } + + line, err = reader.ReadString('\n') + if err != nil { + t.Fatalf("unexpected err %s", err) + } + if line != "sample.slow_thingy:4.000000|ms\n" { + t.Fatalf("bad line %s", line) + } + + conn.Close() + done <- true + }() + s, err := NewStatsiteSink(addr) + if err != nil { + t.Fatalf("bad error") + } + + s.SetGauge([]string{"gauge", "val"}, float32(1)) + s.EmitKey([]string{"key", "other"}, float32(2)) + s.IncrCounter([]string{"counter", "me"}, float32(3)) + s.AddSample([]string{"sample", "slow thingy"}, float32(4)) + + select { + case <-done: + s.Shutdown() + case <-time.After(3 * time.Second): + t.Fatalf("timeout") + } +} diff --git a/Godeps/_workspace/src/github.com/contiv/objmodel/objdb/objdbClient/objdbClient.go b/Godeps/_workspace/src/github.com/contiv/objmodel/objdb/client/client.go similarity index 96% rename from Godeps/_workspace/src/github.com/contiv/objmodel/objdb/objdbClient/objdbClient.go rename to Godeps/_workspace/src/github.com/contiv/objmodel/objdb/client/client.go index 8653060ca..b94241edb 100644 --- a/Godeps/_workspace/src/github.com/contiv/objmodel/objdb/objdbClient/objdbClient.go +++ b/Godeps/_workspace/src/github.com/contiv/objmodel/objdb/client/client.go @@ -1,4 +1,4 @@ -package objdbClient +package client import ( log "github.com/Sirupsen/logrus" diff --git a/Godeps/_workspace/src/github.com/contiv/objmodel/objdb/objdbClient/objdbClient_test.go b/Godeps/_workspace/src/github.com/contiv/objmodel/objdb/client/client_test.go similarity index 99% rename from Godeps/_workspace/src/github.com/contiv/objmodel/objdb/objdbClient/objdbClient_test.go rename to Godeps/_workspace/src/github.com/contiv/objmodel/objdb/client/client_test.go index aab609302..659339029 100644 --- a/Godeps/_workspace/src/github.com/contiv/objmodel/objdb/objdbClient/objdbClient_test.go +++ b/Godeps/_workspace/src/github.com/contiv/objmodel/objdb/client/client_test.go @@ -1,4 +1,4 @@ -package objdbClient +package client import ( "fmt" diff --git a/Godeps/_workspace/src/github.com/contiv/objmodel/objdb/modeldb/modeldb.go b/Godeps/_workspace/src/github.com/contiv/objmodel/objdb/modeldb/modeldb.go index 8f08634c9..d29e714d8 100644 --- a/Godeps/_workspace/src/github.com/contiv/objmodel/objdb/modeldb/modeldb.go +++ b/Godeps/_workspace/src/github.com/contiv/objmodel/objdb/modeldb/modeldb.go @@ -18,7 +18,9 @@ package modeldb // Wrapper for persistently storing object model import ( - "github.com/contiv/objmodel/objdb/objdbClient" + "sync" + + "github.com/contiv/objmodel/objdb/client" log "github.com/Sirupsen/logrus" ) @@ -34,10 +36,14 @@ type ModelObj interface { type Link struct { ObjType string `json:"type,omitempty"` ObjKey string `json:"key,omitempty"` + + mutex sync.Mutex } // AddLink adds a one way link to target object func AddLink(link *Link, obj ModelObj) { + link.mutex.Lock() + defer link.mutex.Unlock() link.ObjType = obj.GetType() link.ObjKey = obj.GetKey() } @@ -77,7 +83,7 @@ func RemoveLinkSet(linkSet *(map[string]Link), obj ModelObj) error { } // persistent database -var cdb = objdbClient.NewClient() +var cdb = client.NewClient() func WriteObj(objType, objKey string, value interface{}) error { key := "/modeldb/" + objType + "/" + objKey diff --git a/Godeps/_workspace/src/github.com/contiv/objmodel/objdb/objdbApi.go b/Godeps/_workspace/src/github.com/contiv/objmodel/objdb/objdb.go similarity index 100% rename from Godeps/_workspace/src/github.com/contiv/objmodel/objdb/objdbApi.go rename to Godeps/_workspace/src/github.com/contiv/objmodel/objdb/objdb.go diff --git a/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/add_child.go b/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/add_child.go new file mode 100644 index 000000000..7122be049 --- /dev/null +++ b/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/add_child.go @@ -0,0 +1,23 @@ +package etcd + +// Add a new directory with a random etcd-generated key under the given path. +func (c *Client) AddChildDir(key string, ttl uint64) (*Response, error) { + raw, err := c.post(key, "", ttl) + + if err != nil { + return nil, err + } + + return raw.Unmarshal() +} + +// Add a new file with a random etcd-generated key under the given path. +func (c *Client) AddChild(key string, value string, ttl uint64) (*Response, error) { + raw, err := c.post(key, value, ttl) + + if err != nil { + return nil, err + } + + return raw.Unmarshal() +} diff --git a/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/add_child_test.go b/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/add_child_test.go new file mode 100644 index 000000000..26223ff1c --- /dev/null +++ b/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/add_child_test.go @@ -0,0 +1,73 @@ +package etcd + +import "testing" + +func TestAddChild(t *testing.T) { + c := NewClient(nil) + defer func() { + c.Delete("fooDir", true) + c.Delete("nonexistentDir", true) + }() + + c.CreateDir("fooDir", 5) + + _, err := c.AddChild("fooDir", "v0", 5) + if err != nil { + t.Fatal(err) + } + + _, err = c.AddChild("fooDir", "v1", 5) + if err != nil { + t.Fatal(err) + } + + resp, err := c.Get("fooDir", true, false) + // The child with v0 should proceed the child with v1 because it's added + // earlier, so it should have a lower key. + if !(len(resp.Node.Nodes) == 2 && (resp.Node.Nodes[0].Value == "v0" && resp.Node.Nodes[1].Value == "v1")) { + t.Fatalf("AddChild 1 failed. There should be two chlidren whose values are v0 and v1, respectively."+ + " The response was: %#v", resp) + } + + // Creating a child under a nonexistent directory should succeed. + // The directory should be created. + resp, err = c.AddChild("nonexistentDir", "foo", 5) + if err != nil { + t.Fatal(err) + } +} + +func TestAddChildDir(t *testing.T) { + c := NewClient(nil) + defer func() { + c.Delete("fooDir", true) + c.Delete("nonexistentDir", true) + }() + + c.CreateDir("fooDir", 5) + + _, err := c.AddChildDir("fooDir", 5) + if err != nil { + t.Fatal(err) + } + + _, err = c.AddChildDir("fooDir", 5) + if err != nil { + t.Fatal(err) + } + + resp, err := c.Get("fooDir", true, false) + // The child with v0 should proceed the child with v1 because it's added + // earlier, so it should have a lower key. + if !(len(resp.Node.Nodes) == 2 && (len(resp.Node.Nodes[0].Nodes) == 0 && len(resp.Node.Nodes[1].Nodes) == 0)) { + t.Fatalf("AddChildDir 1 failed. There should be two chlidren whose values are v0 and v1, respectively."+ + " The response was: %#v", resp) + } + + // Creating a child under a nonexistent directory should succeed. + // The directory should be created. + resp, err = c.AddChildDir("nonexistentDir", 5) + if err != nil { + t.Fatal(err) + } +} diff --git a/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/client.go b/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/client.go new file mode 100644 index 000000000..c6cf3341b --- /dev/null +++ b/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/client.go @@ -0,0 +1,481 @@ +package etcd + +import ( + "crypto/tls" + "crypto/x509" + "encoding/json" + "errors" + "io" + "io/ioutil" + "math/rand" + "net" + "net/http" + "net/url" + "os" + "path" + "strings" + "time" +) + +// See SetConsistency for how to use these constants. +const ( + // Using strings rather than iota because the consistency level + // could be persisted to disk, so it'd be better to use + // human-readable values. + STRONG_CONSISTENCY = "STRONG" + WEAK_CONSISTENCY = "WEAK" +) + +const ( + defaultBufferSize = 10 +) + +func init() { + rand.Seed(int64(time.Now().Nanosecond())) +} + +type Config struct { + CertFile string `json:"certFile"` + KeyFile string `json:"keyFile"` + CaCertFile []string `json:"caCertFiles"` + DialTimeout time.Duration `json:"timeout"` + Consistency string `json:"consistency"` +} + +type credentials struct { + username string + password string +} + +type Client struct { + config Config `json:"config"` + cluster *Cluster `json:"cluster"` + httpClient *http.Client + credentials *credentials + transport *http.Transport + persistence io.Writer + cURLch chan string + // CheckRetry can be used to control the policy for failed requests + // and modify the cluster if needed. + // The client calls it before sending requests again, and + // stops retrying if CheckRetry returns some error. The cases that + // this function needs to handle include no response and unexpected + // http status code of response. + // If CheckRetry is nil, client will call the default one + // `DefaultCheckRetry`. + // Argument cluster is the etcd.Cluster object that these requests have been made on. + // Argument numReqs is the number of http.Requests that have been made so far. + // Argument lastResp is the http.Responses from the last request. + // Argument err is the reason of the failure. + CheckRetry func(cluster *Cluster, numReqs int, + lastResp http.Response, err error) error +} + +// NewClient create a basic client that is configured to be used +// with the given machine list. +func NewClient(machines []string) *Client { + config := Config{ + // default timeout is one second + DialTimeout: time.Second, + Consistency: WEAK_CONSISTENCY, + } + + client := &Client{ + cluster: NewCluster(machines), + config: config, + } + + client.initHTTPClient() + client.saveConfig() + + return client +} + +// NewTLSClient create a basic client with TLS configuration +func NewTLSClient(machines []string, cert, key, caCert string) (*Client, error) { + // overwrite the default machine to use https + if len(machines) == 0 { + machines = []string{"https://127.0.0.1:4001"} + } + + config := Config{ + // default timeout is one second + DialTimeout: time.Second, + Consistency: WEAK_CONSISTENCY, + CertFile: cert, + KeyFile: key, + CaCertFile: make([]string, 0), + } + + client := &Client{ + cluster: NewCluster(machines), + config: config, + } + + err := client.initHTTPSClient(cert, key) + if err != nil { + return nil, err + } + + err = client.AddRootCA(caCert) + + client.saveConfig() + + return client, nil +} + +// NewClientFromFile creates a client from a given file path. +// The given file is expected to use the JSON format. +func NewClientFromFile(fpath string) (*Client, error) { + fi, err := os.Open(fpath) + if err != nil { + return nil, err + } + + defer func() { + if err := fi.Close(); err != nil { + panic(err) + } + }() + + return NewClientFromReader(fi) +} + +// NewClientFromReader creates a Client configured from a given reader. +// The configuration is expected to use the JSON format. +func NewClientFromReader(reader io.Reader) (*Client, error) { + c := new(Client) + + b, err := ioutil.ReadAll(reader) + if err != nil { + return nil, err + } + + err = json.Unmarshal(b, c) + if err != nil { + return nil, err + } + if c.config.CertFile == "" { + c.initHTTPClient() + } else { + err = c.initHTTPSClient(c.config.CertFile, c.config.KeyFile) + } + + if err != nil { + return nil, err + } + + for _, caCert := range c.config.CaCertFile { + if err := c.AddRootCA(caCert); err != nil { + return nil, err + } + } + + return c, nil +} + +// Override the Client's HTTP Transport object +func (c *Client) SetTransport(tr *http.Transport) { + c.httpClient.Transport = tr + c.transport = tr +} + +func (c *Client) SetCredentials(username, password string) { + c.credentials = &credentials{username, password} +} + +func (c *Client) Close() { + c.transport.DisableKeepAlives = true + c.transport.CloseIdleConnections() +} + +// initHTTPClient initializes a HTTP client for etcd client +func (c *Client) initHTTPClient() { + c.transport = &http.Transport{ + Dial: c.dial, + TLSClientConfig: &tls.Config{ + InsecureSkipVerify: true, + }, + } + c.httpClient = &http.Client{Transport: c.transport} +} + +// initHTTPClient initializes a HTTPS client for etcd client +func (c *Client) initHTTPSClient(cert, key string) error { + if cert == "" || key == "" { + return errors.New("Require both cert and key path") + } + + tlsCert, err := tls.LoadX509KeyPair(cert, key) + if err != nil { + return err + } + + tlsConfig := &tls.Config{ + Certificates: []tls.Certificate{tlsCert}, + InsecureSkipVerify: true, + } + + tr := &http.Transport{ + TLSClientConfig: tlsConfig, + Dial: c.dial, + } + + c.httpClient = &http.Client{Transport: tr} + return nil +} + +// SetPersistence sets a writer to which the config will be +// written every time it's changed. +func (c *Client) SetPersistence(writer io.Writer) { + c.persistence = writer +} + +// SetConsistency changes the consistency level of the client. +// +// When consistency is set to STRONG_CONSISTENCY, all requests, +// including GET, are sent to the leader. This means that, assuming +// the absence of leader failures, GET requests are guaranteed to see +// the changes made by previous requests. +// +// When consistency is set to WEAK_CONSISTENCY, other requests +// are still sent to the leader, but GET requests are sent to a +// random server from the server pool. This reduces the read +// load on the leader, but it's not guaranteed that the GET requests +// will see changes made by previous requests (they might have not +// yet been committed on non-leader servers). +func (c *Client) SetConsistency(consistency string) error { + if !(consistency == STRONG_CONSISTENCY || consistency == WEAK_CONSISTENCY) { + return errors.New("The argument must be either STRONG_CONSISTENCY or WEAK_CONSISTENCY.") + } + c.config.Consistency = consistency + return nil +} + +// Sets the DialTimeout value +func (c *Client) SetDialTimeout(d time.Duration) { + c.config.DialTimeout = d +} + +// AddRootCA adds a root CA cert for the etcd client +func (c *Client) AddRootCA(caCert string) error { + if c.httpClient == nil { + return errors.New("Client has not been initialized yet!") + } + + certBytes, err := ioutil.ReadFile(caCert) + if err != nil { + return err + } + + tr, ok := c.httpClient.Transport.(*http.Transport) + + if !ok { + panic("AddRootCA(): Transport type assert should not fail") + } + + if tr.TLSClientConfig.RootCAs == nil { + caCertPool := x509.NewCertPool() + ok = caCertPool.AppendCertsFromPEM(certBytes) + if ok { + tr.TLSClientConfig.RootCAs = caCertPool + } + tr.TLSClientConfig.InsecureSkipVerify = false + } else { + ok = tr.TLSClientConfig.RootCAs.AppendCertsFromPEM(certBytes) + } + + if !ok { + err = errors.New("Unable to load caCert") + } + + c.config.CaCertFile = append(c.config.CaCertFile, caCert) + c.saveConfig() + + return err +} + +// SetCluster updates cluster information using the given machine list. +func (c *Client) SetCluster(machines []string) bool { + success := c.internalSyncCluster(machines) + return success +} + +func (c *Client) GetCluster() []string { + return c.cluster.Machines +} + +// SyncCluster updates the cluster information using the internal machine list. +func (c *Client) SyncCluster() bool { + return c.internalSyncCluster(c.cluster.Machines) +} + +// internalSyncCluster syncs cluster information using the given machine list. +func (c *Client) internalSyncCluster(machines []string) bool { + for _, machine := range machines { + httpPath := c.createHttpPath(machine, path.Join(version, "members")) + resp, err := c.httpClient.Get(httpPath) + if err != nil { + // try another machine in the cluster + continue + } + + if resp.StatusCode != http.StatusOK { // fall-back to old endpoint + httpPath := c.createHttpPath(machine, path.Join(version, "machines")) + resp, err := c.httpClient.Get(httpPath) + if err != nil { + // try another machine in the cluster + continue + } + b, err := ioutil.ReadAll(resp.Body) + resp.Body.Close() + if err != nil { + // try another machine in the cluster + continue + } + // update Machines List + c.cluster.updateFromStr(string(b)) + } else { + b, err := ioutil.ReadAll(resp.Body) + resp.Body.Close() + if err != nil { + // try another machine in the cluster + continue + } + + var mCollection memberCollection + if err := json.Unmarshal(b, &mCollection); err != nil { + // try another machine + continue + } + + urls := make([]string, 0) + for _, m := range mCollection { + urls = append(urls, m.ClientURLs...) + } + + // update Machines List + c.cluster.updateFromStr(strings.Join(urls, ",")) + } + + logger.Debug("sync.machines ", c.cluster.Machines) + c.saveConfig() + return true + } + + return false +} + +// createHttpPath creates a complete HTTP URL. +// serverName should contain both the host name and a port number, if any. +func (c *Client) createHttpPath(serverName string, _path string) string { + u, err := url.Parse(serverName) + if err != nil { + panic(err) + } + + u.Path = path.Join(u.Path, _path) + + if u.Scheme == "" { + u.Scheme = "http" + } + return u.String() +} + +// dial attempts to open a TCP connection to the provided address, explicitly +// enabling keep-alives with a one-second interval. +func (c *Client) dial(network, addr string) (net.Conn, error) { + conn, err := net.DialTimeout(network, addr, c.config.DialTimeout) + if err != nil { + return nil, err + } + + tcpConn, ok := conn.(*net.TCPConn) + if !ok { + return nil, errors.New("Failed type-assertion of net.Conn as *net.TCPConn") + } + + // Keep TCP alive to check whether or not the remote machine is down + if err = tcpConn.SetKeepAlive(true); err != nil { + return nil, err + } + + if err = tcpConn.SetKeepAlivePeriod(time.Second); err != nil { + return nil, err + } + + return tcpConn, nil +} + +func (c *Client) OpenCURL() { + c.cURLch = make(chan string, defaultBufferSize) +} + +func (c *Client) CloseCURL() { + c.cURLch = nil +} + +func (c *Client) sendCURL(command string) { + go func() { + select { + case c.cURLch <- command: + default: + } + }() +} + +func (c *Client) RecvCURL() string { + return <-c.cURLch +} + +// saveConfig saves the current config using c.persistence. +func (c *Client) saveConfig() error { + if c.persistence != nil { + b, err := json.Marshal(c) + if err != nil { + return err + } + + _, err = c.persistence.Write(b) + if err != nil { + return err + } + } + + return nil +} + +// MarshalJSON implements the Marshaller interface +// as defined by the standard JSON package. +func (c *Client) MarshalJSON() ([]byte, error) { + b, err := json.Marshal(struct { + Config Config `json:"config"` + Cluster *Cluster `json:"cluster"` + }{ + Config: c.config, + Cluster: c.cluster, + }) + + if err != nil { + return nil, err + } + + return b, nil +} + +// UnmarshalJSON implements the Unmarshaller interface +// as defined by the standard JSON package. +func (c *Client) UnmarshalJSON(b []byte) error { + temp := struct { + Config Config `json:"config"` + Cluster *Cluster `json:"cluster"` + }{} + err := json.Unmarshal(b, &temp) + if err != nil { + return err + } + + c.cluster = temp.Cluster + c.config = temp.Config + return nil +} diff --git a/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/client_test.go b/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/client_test.go new file mode 100644 index 000000000..4720d8d69 --- /dev/null +++ b/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/client_test.go @@ -0,0 +1,108 @@ +package etcd + +import ( + "encoding/json" + "fmt" + "net" + "net/url" + "os" + "testing" +) + +// To pass this test, we need to create a cluster of 3 machines +// The server should be listening on localhost:4001, 4002, 4003 +func TestSync(t *testing.T) { + fmt.Println("Make sure there are three nodes at 0.0.0.0:4001-4003") + + // Explicit trailing slash to ensure this doesn't reproduce: + // https://github.com/coreos/go-etcd/issues/82 + c := NewClient([]string{"http://127.0.0.1:4001/"}) + + success := c.SyncCluster() + if !success { + t.Fatal("cannot sync machines") + } + + for _, m := range c.GetCluster() { + u, err := url.Parse(m) + if err != nil { + t.Fatal(err) + } + if u.Scheme != "http" { + t.Fatal("scheme must be http") + } + + host, _, err := net.SplitHostPort(u.Host) + if err != nil { + t.Fatal(err) + } + if host != "localhost" { + t.Fatal("Host must be localhost") + } + } + + badMachines := []string{"abc", "edef"} + + success = c.SetCluster(badMachines) + + if success { + t.Fatal("should not sync on bad machines") + } + + goodMachines := []string{"127.0.0.1:4002"} + + success = c.SetCluster(goodMachines) + + if !success { + t.Fatal("cannot sync machines") + } else { + fmt.Println(c.cluster.Machines) + } + +} + +func TestPersistence(t *testing.T) { + c := NewClient(nil) + c.SyncCluster() + + fo, err := os.Create("config.json") + if err != nil { + t.Fatal(err) + } + defer func() { + if err := fo.Close(); err != nil { + panic(err) + } + }() + + c.SetPersistence(fo) + err = c.saveConfig() + if err != nil { + t.Fatal(err) + } + + c2, err := NewClientFromFile("config.json") + if err != nil { + t.Fatal(err) + } + + // Verify that the two clients have the same config + b1, _ := json.Marshal(c) + b2, _ := json.Marshal(c2) + + if string(b1) != string(b2) { + t.Fatalf("The two configs should be equal!") + } +} + +func TestClientRetry(t *testing.T) { + c := NewClient([]string{"http://strange", "http://127.0.0.1:4001"}) + // use first endpoint as the picked url + c.cluster.picked = 0 + if _, err := c.Set("foo", "bar", 5); err != nil { + t.Fatal(err) + } + if _, err := c.Delete("foo", true); err != nil { + t.Fatal(err) + } +} diff --git a/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/cluster.go b/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/cluster.go new file mode 100644 index 000000000..1ad3e155b --- /dev/null +++ b/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/cluster.go @@ -0,0 +1,37 @@ +package etcd + +import ( + "math/rand" + "strings" +) + +type Cluster struct { + Leader string `json:"leader"` + Machines []string `json:"machines"` + picked int +} + +func NewCluster(machines []string) *Cluster { + // if an empty slice was sent in then just assume HTTP 4001 on localhost + if len(machines) == 0 { + machines = []string{"http://127.0.0.1:4001"} + } + + // default leader and machines + return &Cluster{ + Leader: "", + Machines: machines, + picked: rand.Intn(len(machines)), + } +} + +func (cl *Cluster) failure() { cl.picked = rand.Intn(len(cl.Machines)) } +func (cl *Cluster) pick() string { return cl.Machines[cl.picked] } + +func (cl *Cluster) updateFromStr(machines string) { + cl.Machines = strings.Split(machines, ",") + for i := range cl.Machines { + cl.Machines[i] = strings.TrimSpace(cl.Machines[i]) + } + cl.picked = rand.Intn(len(cl.Machines)) +} diff --git a/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/compare_and_delete.go b/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/compare_and_delete.go new file mode 100644 index 000000000..11131bb76 --- /dev/null +++ b/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/compare_and_delete.go @@ -0,0 +1,34 @@ +package etcd + +import "fmt" + +func (c *Client) CompareAndDelete(key string, prevValue string, prevIndex uint64) (*Response, error) { + raw, err := c.RawCompareAndDelete(key, prevValue, prevIndex) + if err != nil { + return nil, err + } + + return raw.Unmarshal() +} + +func (c *Client) RawCompareAndDelete(key string, prevValue string, prevIndex uint64) (*RawResponse, error) { + if prevValue == "" && prevIndex == 0 { + return nil, fmt.Errorf("You must give either prevValue or prevIndex.") + } + + options := Options{} + if prevValue != "" { + options["prevValue"] = prevValue + } + if prevIndex != 0 { + options["prevIndex"] = prevIndex + } + + raw, err := c.delete(key, options) + + if err != nil { + return nil, err + } + + return raw, err +} diff --git a/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/compare_and_delete_test.go b/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/compare_and_delete_test.go new file mode 100644 index 000000000..223e50f29 --- /dev/null +++ b/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/compare_and_delete_test.go @@ -0,0 +1,46 @@ +package etcd + +import ( + "testing" +) + +func TestCompareAndDelete(t *testing.T) { + c := NewClient(nil) + defer func() { + c.Delete("foo", true) + }() + + c.Set("foo", "bar", 5) + + // This should succeed an correct prevValue + resp, err := c.CompareAndDelete("foo", "bar", 0) + if err != nil { + t.Fatal(err) + } + if !(resp.PrevNode.Value == "bar" && resp.PrevNode.Key == "/foo" && resp.PrevNode.TTL == 5) { + t.Fatalf("CompareAndDelete 1 prevNode failed: %#v", resp) + } + + resp, _ = c.Set("foo", "bar", 5) + // This should fail because it gives an incorrect prevValue + _, err = c.CompareAndDelete("foo", "xxx", 0) + if err == nil { + t.Fatalf("CompareAndDelete 2 should have failed. The response is: %#v", resp) + } + + // This should succeed because it gives an correct prevIndex + resp, err = c.CompareAndDelete("foo", "", resp.Node.ModifiedIndex) + if err != nil { + t.Fatal(err) + } + if !(resp.PrevNode.Value == "bar" && resp.PrevNode.Key == "/foo" && resp.PrevNode.TTL == 5) { + t.Fatalf("CompareAndSwap 3 prevNode failed: %#v", resp) + } + + c.Set("foo", "bar", 5) + // This should fail because it gives an incorrect prevIndex + resp, err = c.CompareAndDelete("foo", "", 29817514) + if err == nil { + t.Fatalf("CompareAndDelete 4 should have failed. The response is: %#v", resp) + } +} diff --git a/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/compare_and_swap.go b/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/compare_and_swap.go new file mode 100644 index 000000000..bb4f90643 --- /dev/null +++ b/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/compare_and_swap.go @@ -0,0 +1,36 @@ +package etcd + +import "fmt" + +func (c *Client) CompareAndSwap(key string, value string, ttl uint64, + prevValue string, prevIndex uint64) (*Response, error) { + raw, err := c.RawCompareAndSwap(key, value, ttl, prevValue, prevIndex) + if err != nil { + return nil, err + } + + return raw.Unmarshal() +} + +func (c *Client) RawCompareAndSwap(key string, value string, ttl uint64, + prevValue string, prevIndex uint64) (*RawResponse, error) { + if prevValue == "" && prevIndex == 0 { + return nil, fmt.Errorf("You must give either prevValue or prevIndex.") + } + + options := Options{} + if prevValue != "" { + options["prevValue"] = prevValue + } + if prevIndex != 0 { + options["prevIndex"] = prevIndex + } + + raw, err := c.put(key, value, ttl, options) + + if err != nil { + return nil, err + } + + return raw, err +} diff --git a/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/compare_and_swap_test.go b/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/compare_and_swap_test.go new file mode 100644 index 000000000..14a1b00f5 --- /dev/null +++ b/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/compare_and_swap_test.go @@ -0,0 +1,57 @@ +package etcd + +import ( + "testing" +) + +func TestCompareAndSwap(t *testing.T) { + c := NewClient(nil) + defer func() { + c.Delete("foo", true) + }() + + c.Set("foo", "bar", 5) + + // This should succeed + resp, err := c.CompareAndSwap("foo", "bar2", 5, "bar", 0) + if err != nil { + t.Fatal(err) + } + if !(resp.Node.Value == "bar2" && resp.Node.Key == "/foo" && resp.Node.TTL == 5) { + t.Fatalf("CompareAndSwap 1 failed: %#v", resp) + } + + if !(resp.PrevNode.Value == "bar" && resp.PrevNode.Key == "/foo" && resp.PrevNode.TTL == 5) { + t.Fatalf("CompareAndSwap 1 prevNode failed: %#v", resp) + } + + // This should fail because it gives an incorrect prevValue + resp, err = c.CompareAndSwap("foo", "bar3", 5, "xxx", 0) + if err == nil { + t.Fatalf("CompareAndSwap 2 should have failed. The response is: %#v", resp) + } + + resp, err = c.Set("foo", "bar", 5) + if err != nil { + t.Fatal(err) + } + + // This should succeed + resp, err = c.CompareAndSwap("foo", "bar2", 5, "", resp.Node.ModifiedIndex) + if err != nil { + t.Fatal(err) + } + if !(resp.Node.Value == "bar2" && resp.Node.Key == "/foo" && resp.Node.TTL == 5) { + t.Fatalf("CompareAndSwap 3 failed: %#v", resp) + } + + if !(resp.PrevNode.Value == "bar" && resp.PrevNode.Key == "/foo" && resp.PrevNode.TTL == 5) { + t.Fatalf("CompareAndSwap 3 prevNode failed: %#v", resp) + } + + // This should fail because it gives an incorrect prevIndex + resp, err = c.CompareAndSwap("foo", "bar3", 5, "", 29817514) + if err == nil { + t.Fatalf("CompareAndSwap 4 should have failed. The response is: %#v", resp) + } +} diff --git a/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/debug.go b/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/debug.go new file mode 100644 index 000000000..0f777886b --- /dev/null +++ b/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/debug.go @@ -0,0 +1,55 @@ +package etcd + +import ( + "fmt" + "io/ioutil" + "log" + "strings" +) + +var logger *etcdLogger + +func SetLogger(l *log.Logger) { + logger = &etcdLogger{l} +} + +func GetLogger() *log.Logger { + return logger.log +} + +type etcdLogger struct { + log *log.Logger +} + +func (p *etcdLogger) Debug(args ...interface{}) { + msg := "DEBUG: " + fmt.Sprint(args...) + p.log.Println(msg) +} + +func (p *etcdLogger) Debugf(f string, args ...interface{}) { + msg := "DEBUG: " + fmt.Sprintf(f, args...) + // Append newline if necessary + if !strings.HasSuffix(msg, "\n") { + msg = msg + "\n" + } + p.log.Print(msg) +} + +func (p *etcdLogger) Warning(args ...interface{}) { + msg := "WARNING: " + fmt.Sprint(args...) + p.log.Println(msg) +} + +func (p *etcdLogger) Warningf(f string, args ...interface{}) { + msg := "WARNING: " + fmt.Sprintf(f, args...) + // Append newline if necessary + if !strings.HasSuffix(msg, "\n") { + msg = msg + "\n" + } + p.log.Print(msg) +} + +func init() { + // Default logger uses the go default log. + SetLogger(log.New(ioutil.Discard, "go-etcd", log.LstdFlags)) +} diff --git a/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/debug_test.go b/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/debug_test.go new file mode 100644 index 000000000..97f6d1110 --- /dev/null +++ b/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/debug_test.go @@ -0,0 +1,28 @@ +package etcd + +import ( + "testing" +) + +type Foo struct{} +type Bar struct { + one string + two int +} + +// Tests that logs don't panic with arbitrary interfaces +func TestDebug(t *testing.T) { + f := &Foo{} + b := &Bar{"asfd", 3} + for _, test := range []interface{}{ + 1234, + "asdf", + f, + b, + } { + logger.Debug(test) + logger.Debugf("something, %s", test) + logger.Warning(test) + logger.Warningf("something, %s", test) + } +} diff --git a/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/delete.go b/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/delete.go new file mode 100644 index 000000000..b37accd7d --- /dev/null +++ b/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/delete.go @@ -0,0 +1,40 @@ +package etcd + +// Delete deletes the given key. +// +// When recursive set to false, if the key points to a +// directory the method will fail. +// +// When recursive set to true, if the key points to a file, +// the file will be deleted; if the key points to a directory, +// then everything under the directory (including all child directories) +// will be deleted. +func (c *Client) Delete(key string, recursive bool) (*Response, error) { + raw, err := c.RawDelete(key, recursive, false) + + if err != nil { + return nil, err + } + + return raw.Unmarshal() +} + +// DeleteDir deletes an empty directory or a key value pair +func (c *Client) DeleteDir(key string) (*Response, error) { + raw, err := c.RawDelete(key, false, true) + + if err != nil { + return nil, err + } + + return raw.Unmarshal() +} + +func (c *Client) RawDelete(key string, recursive bool, dir bool) (*RawResponse, error) { + ops := Options{ + "recursive": recursive, + "dir": dir, + } + + return c.delete(key, ops) +} diff --git a/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/delete_test.go b/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/delete_test.go new file mode 100644 index 000000000..590497155 --- /dev/null +++ b/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/delete_test.go @@ -0,0 +1,81 @@ +package etcd + +import ( + "testing" +) + +func TestDelete(t *testing.T) { + c := NewClient(nil) + defer func() { + c.Delete("foo", true) + }() + + c.Set("foo", "bar", 5) + resp, err := c.Delete("foo", false) + if err != nil { + t.Fatal(err) + } + + if !(resp.Node.Value == "") { + t.Fatalf("Delete failed with %s", resp.Node.Value) + } + + if !(resp.PrevNode.Value == "bar") { + t.Fatalf("Delete PrevNode failed with %s", resp.Node.Value) + } + + resp, err = c.Delete("foo", false) + if err == nil { + t.Fatalf("Delete should have failed because the key foo did not exist. "+ + "The response was: %v", resp) + } +} + +func TestDeleteAll(t *testing.T) { + c := NewClient(nil) + defer func() { + c.Delete("foo", true) + c.Delete("fooDir", true) + }() + + c.SetDir("foo", 5) + // test delete an empty dir + resp, err := c.DeleteDir("foo") + if err != nil { + t.Fatal(err) + } + + if !(resp.Node.Value == "") { + t.Fatalf("DeleteAll 1 failed: %#v", resp) + } + + if !(resp.PrevNode.Dir == true && resp.PrevNode.Value == "") { + t.Fatalf("DeleteAll 1 PrevNode failed: %#v", resp) + } + + c.CreateDir("fooDir", 5) + c.Set("fooDir/foo", "bar", 5) + _, err = c.DeleteDir("fooDir") + if err == nil { + t.Fatal("should not able to delete a non-empty dir with deletedir") + } + + resp, err = c.Delete("fooDir", true) + if err != nil { + t.Fatal(err) + } + + if !(resp.Node.Value == "") { + t.Fatalf("DeleteAll 2 failed: %#v", resp) + } + + if !(resp.PrevNode.Dir == true && resp.PrevNode.Value == "") { + t.Fatalf("DeleteAll 2 PrevNode failed: %#v", resp) + } + + resp, err = c.Delete("foo", true) + if err == nil { + t.Fatalf("DeleteAll should have failed because the key foo did not exist. "+ + "The response was: %v", resp) + } +} diff --git a/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/error.go b/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/error.go new file mode 100644 index 000000000..66dca54b5 --- /dev/null +++ b/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/error.go @@ -0,0 +1,49 @@ +package etcd + +import ( + "encoding/json" + "fmt" +) + +const ( + ErrCodeEtcdNotReachable = 501 + ErrCodeUnhandledHTTPStatus = 502 +) + +var ( + errorMap = map[int]string{ + ErrCodeEtcdNotReachable: "All the given peers are not reachable", + } +) + +type EtcdError struct { + ErrorCode int `json:"errorCode"` + Message string `json:"message"` + Cause string `json:"cause,omitempty"` + Index uint64 `json:"index"` +} + +func (e EtcdError) Error() string { + return fmt.Sprintf("%v: %v (%v) [%v]", e.ErrorCode, e.Message, e.Cause, e.Index) +} + +func newError(errorCode int, cause string, index uint64) *EtcdError { + return &EtcdError{ + ErrorCode: errorCode, + Message: errorMap[errorCode], + Cause: cause, + Index: index, + } +} + +func handleError(b []byte) error { + etcdErr := new(EtcdError) + + err := json.Unmarshal(b, etcdErr) + if err != nil { + logger.Warningf("cannot unmarshal etcd error: %v", err) + return err + } + + return etcdErr +} diff --git a/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/get.go b/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/get.go new file mode 100644 index 000000000..09fe641c2 --- /dev/null +++ b/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/get.go @@ -0,0 +1,32 @@ +package etcd + +// Get gets the file or directory associated with the given key. +// If the key points to a directory, files and directories under +// it will be returned in sorted or unsorted order, depending on +// the sort flag. +// If recursive is set to false, contents under child directories +// will not be returned. +// If recursive is set to true, all the contents will be returned. +func (c *Client) Get(key string, sort, recursive bool) (*Response, error) { + raw, err := c.RawGet(key, sort, recursive) + + if err != nil { + return nil, err + } + + return raw.Unmarshal() +} + +func (c *Client) RawGet(key string, sort, recursive bool) (*RawResponse, error) { + var q bool + if c.config.Consistency == STRONG_CONSISTENCY { + q = true + } + ops := Options{ + "recursive": recursive, + "sorted": sort, + "quorum": q, + } + + return c.get(key, ops) +} diff --git a/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/get_test.go b/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/get_test.go new file mode 100644 index 000000000..279c4e26f --- /dev/null +++ b/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/get_test.go @@ -0,0 +1,131 @@ +package etcd + +import ( + "reflect" + "testing" +) + +// cleanNode scrubs Expiration, ModifiedIndex and CreatedIndex of a node. +func cleanNode(n *Node) { + n.Expiration = nil + n.ModifiedIndex = 0 + n.CreatedIndex = 0 +} + +// cleanResult scrubs a result object two levels deep of Expiration, +// ModifiedIndex and CreatedIndex. +func cleanResult(result *Response) { + // TODO(philips): make this recursive. + cleanNode(result.Node) + for i, _ := range result.Node.Nodes { + cleanNode(result.Node.Nodes[i]) + for j, _ := range result.Node.Nodes[i].Nodes { + cleanNode(result.Node.Nodes[i].Nodes[j]) + } + } +} + +func TestGet(t *testing.T) { + c := NewClient(nil) + defer func() { + c.Delete("foo", true) + }() + + c.Set("foo", "bar", 5) + + result, err := c.Get("foo", false, false) + + if err != nil { + t.Fatal(err) + } + + if result.Node.Key != "/foo" || result.Node.Value != "bar" { + t.Fatalf("Get failed with %s %s %v", result.Node.Key, result.Node.Value, result.Node.TTL) + } + + result, err = c.Get("goo", false, false) + if err == nil { + t.Fatalf("should not be able to get non-exist key") + } +} + +func TestGetAll(t *testing.T) { + c := NewClient(nil) + defer func() { + c.Delete("fooDir", true) + }() + + c.CreateDir("fooDir", 5) + c.Set("fooDir/k0", "v0", 5) + c.Set("fooDir/k1", "v1", 5) + + // Return kv-pairs in sorted order + result, err := c.Get("fooDir", true, false) + + if err != nil { + t.Fatal(err) + } + + expected := Nodes{ + &Node{ + Key: "/fooDir/k0", + Value: "v0", + TTL: 5, + }, + &Node{ + Key: "/fooDir/k1", + Value: "v1", + TTL: 5, + }, + } + + cleanResult(result) + + if !reflect.DeepEqual(result.Node.Nodes, expected) { + t.Fatalf("(actual) %v != (expected) %v", result.Node.Nodes, expected) + } + + // Test the `recursive` option + c.CreateDir("fooDir/childDir", 5) + c.Set("fooDir/childDir/k2", "v2", 5) + + // Return kv-pairs in sorted order + result, err = c.Get("fooDir", true, true) + + cleanResult(result) + + if err != nil { + t.Fatal(err) + } + + expected = Nodes{ + &Node{ + Key: "/fooDir/childDir", + Dir: true, + Nodes: Nodes{ + &Node{ + Key: "/fooDir/childDir/k2", + Value: "v2", + TTL: 5, + }, + }, + TTL: 5, + }, + &Node{ + Key: "/fooDir/k0", + Value: "v0", + TTL: 5, + }, + &Node{ + Key: "/fooDir/k1", + Value: "v1", + TTL: 5, + }, + } + + cleanResult(result) + + if !reflect.DeepEqual(result.Node.Nodes, expected) { + t.Fatalf("(actual) %v != (expected) %v", result.Node.Nodes, expected) + } +} diff --git a/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/member.go b/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/member.go new file mode 100644 index 000000000..5b13b28e1 --- /dev/null +++ b/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/member.go @@ -0,0 +1,30 @@ +package etcd + +import "encoding/json" + +type Member struct { + ID string `json:"id"` + Name string `json:"name"` + PeerURLs []string `json:"peerURLs"` + ClientURLs []string `json:"clientURLs"` +} + +type memberCollection []Member + +func (c *memberCollection) UnmarshalJSON(data []byte) error { + d := struct { + Members []Member + }{} + + if err := json.Unmarshal(data, &d); err != nil { + return err + } + + if d.Members == nil { + *c = make([]Member, 0) + return nil + } + + *c = d.Members + return nil +} diff --git a/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/member_test.go b/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/member_test.go new file mode 100644 index 000000000..53ebdd4bf --- /dev/null +++ b/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/member_test.go @@ -0,0 +1,71 @@ +package etcd + +import ( + "encoding/json" + "reflect" + "testing" +) + +func TestMemberCollectionUnmarshal(t *testing.T) { + tests := []struct { + body []byte + want memberCollection + }{ + { + body: []byte(`{"members":[]}`), + want: memberCollection([]Member{}), + }, + { + body: []byte(`{"members":[{"id":"2745e2525fce8fe","peerURLs":["http://127.0.0.1:7003"],"name":"node3","clientURLs":["http://127.0.0.1:4003"]},{"id":"42134f434382925","peerURLs":["http://127.0.0.1:2380","http://127.0.0.1:7001"],"name":"node1","clientURLs":["http://127.0.0.1:2379","http://127.0.0.1:4001"]},{"id":"94088180e21eb87b","peerURLs":["http://127.0.0.1:7002"],"name":"node2","clientURLs":["http://127.0.0.1:4002"]}]}`), + want: memberCollection( + []Member{ + { + ID: "2745e2525fce8fe", + Name: "node3", + PeerURLs: []string{ + "http://127.0.0.1:7003", + }, + ClientURLs: []string{ + "http://127.0.0.1:4003", + }, + }, + { + ID: "42134f434382925", + Name: "node1", + PeerURLs: []string{ + "http://127.0.0.1:2380", + "http://127.0.0.1:7001", + }, + ClientURLs: []string{ + "http://127.0.0.1:2379", + "http://127.0.0.1:4001", + }, + }, + { + ID: "94088180e21eb87b", + Name: "node2", + PeerURLs: []string{ + "http://127.0.0.1:7002", + }, + ClientURLs: []string{ + "http://127.0.0.1:4002", + }, + }, + }, + ), + }, + } + + for i, tt := range tests { + var got memberCollection + err := json.Unmarshal(tt.body, &got) + if err != nil { + t.Errorf("#%d: unexpected error: %v", i, err) + continue + } + + if !reflect.DeepEqual(tt.want, got) { + t.Errorf("#%d: incorrect output: want=%#v, got=%#v", i, tt.want, got) + } + } +} diff --git a/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/options.go b/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/options.go new file mode 100644 index 000000000..d21c96f08 --- /dev/null +++ b/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/options.go @@ -0,0 +1,72 @@ +package etcd + +import ( + "fmt" + "net/url" + "reflect" +) + +type Options map[string]interface{} + +// An internally-used data structure that represents a mapping +// between valid options and their kinds +type validOptions map[string]reflect.Kind + +// Valid options for GET, PUT, POST, DELETE +// Using CAPITALIZED_UNDERSCORE to emphasize that these +// values are meant to be used as constants. +var ( + VALID_GET_OPTIONS = validOptions{ + "recursive": reflect.Bool, + "quorum": reflect.Bool, + "sorted": reflect.Bool, + "wait": reflect.Bool, + "waitIndex": reflect.Uint64, + } + + VALID_PUT_OPTIONS = validOptions{ + "prevValue": reflect.String, + "prevIndex": reflect.Uint64, + "prevExist": reflect.Bool, + "dir": reflect.Bool, + } + + VALID_POST_OPTIONS = validOptions{} + + VALID_DELETE_OPTIONS = validOptions{ + "recursive": reflect.Bool, + "dir": reflect.Bool, + "prevValue": reflect.String, + "prevIndex": reflect.Uint64, + } +) + +// Convert options to a string of HTML parameters +func (ops Options) toParameters(validOps validOptions) (string, error) { + p := "?" + values := url.Values{} + + if ops == nil { + return "", nil + } + + for k, v := range ops { + // Check if the given option is valid (that it exists) + kind := validOps[k] + if kind == reflect.Invalid { + return "", fmt.Errorf("Invalid option: %v", k) + } + + // Check if the given option is of the valid type + t := reflect.TypeOf(v) + if kind != t.Kind() { + return "", fmt.Errorf("Option %s should be of %v kind, not of %v kind.", + k, kind, t.Kind()) + } + + values.Set(k, fmt.Sprintf("%v", v)) + } + + p += values.Encode() + return p, nil +} diff --git a/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/requests.go b/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/requests.go new file mode 100644 index 000000000..3c3f436be --- /dev/null +++ b/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/requests.go @@ -0,0 +1,403 @@ +package etcd + +import ( + "errors" + "fmt" + "io" + "io/ioutil" + "net/http" + "net/url" + "path" + "strings" + "sync" + "time" +) + +// Errors introduced by handling requests +var ( + ErrRequestCancelled = errors.New("sending request is cancelled") +) + +type RawRequest struct { + Method string + RelativePath string + Values url.Values + Cancel <-chan bool +} + +// NewRawRequest returns a new RawRequest +func NewRawRequest(method, relativePath string, values url.Values, cancel <-chan bool) *RawRequest { + return &RawRequest{ + Method: method, + RelativePath: relativePath, + Values: values, + Cancel: cancel, + } +} + +// getCancelable issues a cancelable GET request +func (c *Client) getCancelable(key string, options Options, + cancel <-chan bool) (*RawResponse, error) { + logger.Debugf("get %s [%s]", key, c.cluster.pick()) + p := keyToPath(key) + + str, err := options.toParameters(VALID_GET_OPTIONS) + if err != nil { + return nil, err + } + p += str + + req := NewRawRequest("GET", p, nil, cancel) + resp, err := c.SendRequest(req) + + if err != nil { + return nil, err + } + + return resp, nil +} + +// get issues a GET request +func (c *Client) get(key string, options Options) (*RawResponse, error) { + return c.getCancelable(key, options, nil) +} + +// put issues a PUT request +func (c *Client) put(key string, value string, ttl uint64, + options Options) (*RawResponse, error) { + + logger.Debugf("put %s, %s, ttl: %d, [%s]", key, value, ttl, c.cluster.pick()) + p := keyToPath(key) + + str, err := options.toParameters(VALID_PUT_OPTIONS) + if err != nil { + return nil, err + } + p += str + + req := NewRawRequest("PUT", p, buildValues(value, ttl), nil) + resp, err := c.SendRequest(req) + + if err != nil { + return nil, err + } + + return resp, nil +} + +// post issues a POST request +func (c *Client) post(key string, value string, ttl uint64) (*RawResponse, error) { + logger.Debugf("post %s, %s, ttl: %d, [%s]", key, value, ttl, c.cluster.pick()) + p := keyToPath(key) + + req := NewRawRequest("POST", p, buildValues(value, ttl), nil) + resp, err := c.SendRequest(req) + + if err != nil { + return nil, err + } + + return resp, nil +} + +// delete issues a DELETE request +func (c *Client) delete(key string, options Options) (*RawResponse, error) { + logger.Debugf("delete %s [%s]", key, c.cluster.pick()) + p := keyToPath(key) + + str, err := options.toParameters(VALID_DELETE_OPTIONS) + if err != nil { + return nil, err + } + p += str + + req := NewRawRequest("DELETE", p, nil, nil) + resp, err := c.SendRequest(req) + + if err != nil { + return nil, err + } + + return resp, nil +} + +// SendRequest sends a HTTP request and returns a Response as defined by etcd +func (c *Client) SendRequest(rr *RawRequest) (*RawResponse, error) { + var req *http.Request + var resp *http.Response + var httpPath string + var err error + var respBody []byte + + var numReqs = 1 + + checkRetry := c.CheckRetry + if checkRetry == nil { + checkRetry = DefaultCheckRetry + } + + cancelled := make(chan bool, 1) + reqLock := new(sync.Mutex) + + if rr.Cancel != nil { + cancelRoutine := make(chan bool) + defer close(cancelRoutine) + + go func() { + select { + case <-rr.Cancel: + cancelled <- true + logger.Debug("send.request is cancelled") + case <-cancelRoutine: + return + } + + // Repeat canceling request until this thread is stopped + // because we have no idea about whether it succeeds. + for { + reqLock.Lock() + c.httpClient.Transport.(*http.Transport).CancelRequest(req) + reqLock.Unlock() + + select { + case <-time.After(100 * time.Millisecond): + case <-cancelRoutine: + return + } + } + }() + } + + // If we connect to a follower and consistency is required, retry until + // we connect to a leader + sleep := 25 * time.Millisecond + maxSleep := time.Second + + for attempt := 0; ; attempt++ { + if attempt > 0 { + select { + case <-cancelled: + return nil, ErrRequestCancelled + case <-time.After(sleep): + sleep = sleep * 2 + if sleep > maxSleep { + sleep = maxSleep + } + } + } + + logger.Debug("Connecting to etcd: attempt ", attempt+1, " for ", rr.RelativePath) + + // get httpPath if not set + if httpPath == "" { + httpPath = c.getHttpPath(rr.RelativePath) + } + + // Return a cURL command if curlChan is set + if c.cURLch != nil { + command := fmt.Sprintf("curl -X %s %s", rr.Method, httpPath) + for key, value := range rr.Values { + command += fmt.Sprintf(" -d %s=%s", key, value[0]) + } + if c.credentials != nil { + command += fmt.Sprintf(" -u %s", c.credentials.username) + } + c.sendCURL(command) + } + + logger.Debug("send.request.to ", httpPath, " | method ", rr.Method) + + req, err := func() (*http.Request, error) { + reqLock.Lock() + defer reqLock.Unlock() + + if rr.Values == nil { + if req, err = http.NewRequest(rr.Method, httpPath, nil); err != nil { + return nil, err + } + } else { + body := strings.NewReader(rr.Values.Encode()) + if req, err = http.NewRequest(rr.Method, httpPath, body); err != nil { + return nil, err + } + + req.Header.Set("Content-Type", + "application/x-www-form-urlencoded; param=value") + } + return req, nil + }() + + if err != nil { + return nil, err + } + + if c.credentials != nil { + req.SetBasicAuth(c.credentials.username, c.credentials.password) + } + + resp, err = c.httpClient.Do(req) + // clear previous httpPath + httpPath = "" + defer func() { + if resp != nil { + resp.Body.Close() + } + }() + + // If the request was cancelled, return ErrRequestCancelled directly + select { + case <-cancelled: + return nil, ErrRequestCancelled + default: + } + + numReqs++ + + // network error, change a machine! + if err != nil { + logger.Debug("network error: ", err.Error()) + lastResp := http.Response{} + if checkErr := checkRetry(c.cluster, numReqs, lastResp, err); checkErr != nil { + return nil, checkErr + } + + c.cluster.failure() + continue + } + + // if there is no error, it should receive response + logger.Debug("recv.response.from ", httpPath) + + if validHttpStatusCode[resp.StatusCode] { + // try to read byte code and break the loop + respBody, err = ioutil.ReadAll(resp.Body) + if err == nil { + logger.Debug("recv.success ", httpPath) + break + } + // ReadAll error may be caused due to cancel request + select { + case <-cancelled: + return nil, ErrRequestCancelled + default: + } + + if err == io.ErrUnexpectedEOF { + // underlying connection was closed prematurely, probably by timeout + // TODO: empty body or unexpectedEOF can cause http.Transport to get hosed; + // this allows the client to detect that and take evasive action. Need + // to revisit once code.google.com/p/go/issues/detail?id=8648 gets fixed. + respBody = []byte{} + break + } + } + + if resp.StatusCode == http.StatusTemporaryRedirect { + u, err := resp.Location() + + if err != nil { + logger.Warning(err) + } else { + // set httpPath for following redirection + httpPath = u.String() + } + resp.Body.Close() + continue + } + + if checkErr := checkRetry(c.cluster, numReqs, *resp, + errors.New("Unexpected HTTP status code")); checkErr != nil { + return nil, checkErr + } + resp.Body.Close() + } + + r := &RawResponse{ + StatusCode: resp.StatusCode, + Body: respBody, + Header: resp.Header, + } + + return r, nil +} + +// DefaultCheckRetry defines the retrying behaviour for bad HTTP requests +// If we have retried 2 * machine number, stop retrying. +// If status code is InternalServerError, sleep for 200ms. +func DefaultCheckRetry(cluster *Cluster, numReqs int, lastResp http.Response, + err error) error { + + if numReqs > 2*len(cluster.Machines) { + errStr := fmt.Sprintf("failed to propose on members %v twice [last error: %v]", cluster.Machines, err) + return newError(ErrCodeEtcdNotReachable, errStr, 0) + } + + if isEmptyResponse(lastResp) { + // always retry if it failed to get response from one machine + return nil + } + if !shouldRetry(lastResp) { + body := []byte("nil") + if lastResp.Body != nil { + if b, err := ioutil.ReadAll(lastResp.Body); err == nil { + body = b + } + } + errStr := fmt.Sprintf("unhandled http status [%s] with body [%s]", http.StatusText(lastResp.StatusCode), body) + return newError(ErrCodeUnhandledHTTPStatus, errStr, 0) + } + // sleep some time and expect leader election finish + time.Sleep(time.Millisecond * 200) + logger.Warning("bad response status code", lastResp.StatusCode) + return nil +} + +func isEmptyResponse(r http.Response) bool { return r.StatusCode == 0 } + +// shouldRetry returns whether the reponse deserves retry. +func shouldRetry(r http.Response) bool { + // TODO: only retry when the cluster is in leader election + // We cannot do it exactly because etcd doesn't support it well. + return r.StatusCode == http.StatusInternalServerError +} + +func (c *Client) getHttpPath(s ...string) string { + fullPath := c.cluster.pick() + "/" + version + for _, seg := range s { + fullPath = fullPath + "/" + seg + } + return fullPath +} + +// buildValues builds a url.Values map according to the given value and ttl +func buildValues(value string, ttl uint64) url.Values { + v := url.Values{} + + if value != "" { + v.Set("value", value) + } + + if ttl > 0 { + v.Set("ttl", fmt.Sprintf("%v", ttl)) + } + + return v +} + +// convert key string to http path exclude version, including URL escaping +// for example: key[foo] -> path[keys/foo] +// key[/%z] -> path[keys/%25z] +// key[/] -> path[keys/] +func keyToPath(key string) string { + // URL-escape our key, except for slashes + p := strings.Replace(url.QueryEscape(path.Join("keys", key)), "%2F", "/", -1) + + // corner case: if key is "/" or "//" ect + // path join will clear the tailing "/" + // we need to add it back + if p == "keys" { + p = "keys/" + } + + return p +} diff --git a/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/requests_test.go b/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/requests_test.go new file mode 100644 index 000000000..7a2bd190a --- /dev/null +++ b/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/requests_test.go @@ -0,0 +1,22 @@ +package etcd + +import "testing" + +func TestKeyToPath(t *testing.T) { + tests := []struct { + key string + wpath string + }{ + {"", "keys/"}, + {"foo", "keys/foo"}, + {"foo/bar", "keys/foo/bar"}, + {"%z", "keys/%25z"}, + {"/", "keys/"}, + } + for i, tt := range tests { + path := keyToPath(tt.key) + if path != tt.wpath { + t.Errorf("#%d: path = %s, want %s", i, path, tt.wpath) + } + } +} diff --git a/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/response.go b/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/response.go new file mode 100644 index 000000000..1fe9b4e87 --- /dev/null +++ b/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/response.go @@ -0,0 +1,89 @@ +package etcd + +import ( + "encoding/json" + "net/http" + "strconv" + "time" +) + +const ( + rawResponse = iota + normalResponse +) + +type responseType int + +type RawResponse struct { + StatusCode int + Body []byte + Header http.Header +} + +var ( + validHttpStatusCode = map[int]bool{ + http.StatusCreated: true, + http.StatusOK: true, + http.StatusBadRequest: true, + http.StatusNotFound: true, + http.StatusPreconditionFailed: true, + http.StatusForbidden: true, + } +) + +// Unmarshal parses RawResponse and stores the result in Response +func (rr *RawResponse) Unmarshal() (*Response, error) { + if rr.StatusCode != http.StatusOK && rr.StatusCode != http.StatusCreated { + return nil, handleError(rr.Body) + } + + resp := new(Response) + + err := json.Unmarshal(rr.Body, resp) + + if err != nil { + return nil, err + } + + // attach index and term to response + resp.EtcdIndex, _ = strconv.ParseUint(rr.Header.Get("X-Etcd-Index"), 10, 64) + resp.RaftIndex, _ = strconv.ParseUint(rr.Header.Get("X-Raft-Index"), 10, 64) + resp.RaftTerm, _ = strconv.ParseUint(rr.Header.Get("X-Raft-Term"), 10, 64) + + return resp, nil +} + +type Response struct { + Action string `json:"action"` + Node *Node `json:"node"` + PrevNode *Node `json:"prevNode,omitempty"` + EtcdIndex uint64 `json:"etcdIndex"` + RaftIndex uint64 `json:"raftIndex"` + RaftTerm uint64 `json:"raftTerm"` +} + +type Node struct { + Key string `json:"key, omitempty"` + Value string `json:"value,omitempty"` + Dir bool `json:"dir,omitempty"` + Expiration *time.Time `json:"expiration,omitempty"` + TTL int64 `json:"ttl,omitempty"` + Nodes Nodes `json:"nodes,omitempty"` + ModifiedIndex uint64 `json:"modifiedIndex,omitempty"` + CreatedIndex uint64 `json:"createdIndex,omitempty"` +} + +type Nodes []*Node + +// interfaces for sorting +func (ns Nodes) Len() int { + return len(ns) +} + +func (ns Nodes) Less(i, j int) bool { + return ns[i].Key < ns[j].Key +} + +func (ns Nodes) Swap(i, j int) { + ns[i], ns[j] = ns[j], ns[i] +} diff --git a/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/set_curl_chan_test.go b/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/set_curl_chan_test.go new file mode 100644 index 000000000..87c86b830 --- /dev/null +++ b/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/set_curl_chan_test.go @@ -0,0 +1,42 @@ +package etcd + +import ( + "fmt" + "testing" +) + +func TestSetCurlChan(t *testing.T) { + c := NewClient(nil) + c.OpenCURL() + + defer func() { + c.Delete("foo", true) + }() + + _, err := c.Set("foo", "bar", 5) + if err != nil { + t.Fatal(err) + } + + expected := fmt.Sprintf("curl -X PUT %s/v2/keys/foo -d value=bar -d ttl=5", + c.cluster.pick()) + actual := c.RecvCURL() + if expected != actual { + t.Fatalf(`Command "%s" is not equal to expected value "%s"`, + actual, expected) + } + + c.SetConsistency(STRONG_CONSISTENCY) + _, err = c.Get("foo", false, false) + if err != nil { + t.Fatal(err) + } + + expected = fmt.Sprintf("curl -X GET %s/v2/keys/foo?quorum=true&recursive=false&sorted=false", + c.cluster.pick()) + actual = c.RecvCURL() + if expected != actual { + t.Fatalf(`Command "%s" is not equal to expected value "%s"`, + actual, expected) + } +} diff --git a/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/set_update_create.go b/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/set_update_create.go new file mode 100644 index 000000000..e2840cf35 --- /dev/null +++ b/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/set_update_create.go @@ -0,0 +1,137 @@ +package etcd + +// Set sets the given key to the given value. +// It will create a new key value pair or replace the old one. +// It will not replace a existing directory. +func (c *Client) Set(key string, value string, ttl uint64) (*Response, error) { + raw, err := c.RawSet(key, value, ttl) + + if err != nil { + return nil, err + } + + return raw.Unmarshal() +} + +// SetDir sets the given key to a directory. +// It will create a new directory or replace the old key value pair by a directory. +// It will not replace a existing directory. +func (c *Client) SetDir(key string, ttl uint64) (*Response, error) { + raw, err := c.RawSetDir(key, ttl) + + if err != nil { + return nil, err + } + + return raw.Unmarshal() +} + +// CreateDir creates a directory. It succeeds only if +// the given key does not yet exist. +func (c *Client) CreateDir(key string, ttl uint64) (*Response, error) { + raw, err := c.RawCreateDir(key, ttl) + + if err != nil { + return nil, err + } + + return raw.Unmarshal() +} + +// UpdateDir updates the given directory. It succeeds only if the +// given key already exists. +func (c *Client) UpdateDir(key string, ttl uint64) (*Response, error) { + raw, err := c.RawUpdateDir(key, ttl) + + if err != nil { + return nil, err + } + + return raw.Unmarshal() +} + +// Create creates a file with the given value under the given key. It succeeds +// only if the given key does not yet exist. +func (c *Client) Create(key string, value string, ttl uint64) (*Response, error) { + raw, err := c.RawCreate(key, value, ttl) + + if err != nil { + return nil, err + } + + return raw.Unmarshal() +} + +// CreateInOrder creates a file with a key that's guaranteed to be higher than other +// keys in the given directory. It is useful for creating queues. +func (c *Client) CreateInOrder(dir string, value string, ttl uint64) (*Response, error) { + raw, err := c.RawCreateInOrder(dir, value, ttl) + + if err != nil { + return nil, err + } + + return raw.Unmarshal() +} + +// Update updates the given key to the given value. It succeeds only if the +// given key already exists. +func (c *Client) Update(key string, value string, ttl uint64) (*Response, error) { + raw, err := c.RawUpdate(key, value, ttl) + + if err != nil { + return nil, err + } + + return raw.Unmarshal() +} + +func (c *Client) RawUpdateDir(key string, ttl uint64) (*RawResponse, error) { + ops := Options{ + "prevExist": true, + "dir": true, + } + + return c.put(key, "", ttl, ops) +} + +func (c *Client) RawCreateDir(key string, ttl uint64) (*RawResponse, error) { + ops := Options{ + "prevExist": false, + "dir": true, + } + + return c.put(key, "", ttl, ops) +} + +func (c *Client) RawSet(key string, value string, ttl uint64) (*RawResponse, error) { + return c.put(key, value, ttl, nil) +} + +func (c *Client) RawSetDir(key string, ttl uint64) (*RawResponse, error) { + ops := Options{ + "dir": true, + } + + return c.put(key, "", ttl, ops) +} + +func (c *Client) RawUpdate(key string, value string, ttl uint64) (*RawResponse, error) { + ops := Options{ + "prevExist": true, + } + + return c.put(key, value, ttl, ops) +} + +func (c *Client) RawCreate(key string, value string, ttl uint64) (*RawResponse, error) { + ops := Options{ + "prevExist": false, + } + + return c.put(key, value, ttl, ops) +} + +func (c *Client) RawCreateInOrder(dir string, value string, ttl uint64) (*RawResponse, error) { + return c.post(dir, value, ttl) +} diff --git a/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/set_update_create_test.go b/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/set_update_create_test.go new file mode 100644 index 000000000..ced0f06e7 --- /dev/null +++ b/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/set_update_create_test.go @@ -0,0 +1,241 @@ +package etcd + +import ( + "testing" +) + +func TestSet(t *testing.T) { + c := NewClient(nil) + defer func() { + c.Delete("foo", true) + }() + + resp, err := c.Set("foo", "bar", 5) + if err != nil { + t.Fatal(err) + } + if resp.Node.Key != "/foo" || resp.Node.Value != "bar" || resp.Node.TTL != 5 { + t.Fatalf("Set 1 failed: %#v", resp) + } + if resp.PrevNode != nil { + t.Fatalf("Set 1 PrevNode failed: %#v", resp) + } + + resp, err = c.Set("foo", "bar2", 5) + if err != nil { + t.Fatal(err) + } + if !(resp.Node.Key == "/foo" && resp.Node.Value == "bar2" && resp.Node.TTL == 5) { + t.Fatalf("Set 2 failed: %#v", resp) + } + if resp.PrevNode.Key != "/foo" || resp.PrevNode.Value != "bar" || resp.Node.TTL != 5 { + t.Fatalf("Set 2 PrevNode failed: %#v", resp) + } +} + +func TestUpdate(t *testing.T) { + c := NewClient(nil) + defer func() { + c.Delete("foo", true) + c.Delete("nonexistent", true) + }() + + resp, err := c.Set("foo", "bar", 5) + + if err != nil { + t.Fatal(err) + } + + // This should succeed. + resp, err = c.Update("foo", "wakawaka", 5) + if err != nil { + t.Fatal(err) + } + + if !(resp.Action == "update" && resp.Node.Key == "/foo" && resp.Node.TTL == 5) { + t.Fatalf("Update 1 failed: %#v", resp) + } + if !(resp.PrevNode.Key == "/foo" && resp.PrevNode.Value == "bar" && resp.Node.TTL == 5) { + t.Fatalf("Update 1 prevValue failed: %#v", resp) + } + + // This should fail because the key does not exist. + resp, err = c.Update("nonexistent", "whatever", 5) + if err == nil { + t.Fatalf("The key %v did not exist, so the update should have failed."+ + "The response was: %#v", resp.Node.Key, resp) + } +} + +func TestCreate(t *testing.T) { + c := NewClient(nil) + defer func() { + c.Delete("newKey", true) + }() + + newKey := "/newKey" + newValue := "/newValue" + + // This should succeed + resp, err := c.Create(newKey, newValue, 5) + if err != nil { + t.Fatal(err) + } + + if !(resp.Action == "create" && resp.Node.Key == newKey && + resp.Node.Value == newValue && resp.Node.TTL == 5) { + t.Fatalf("Create 1 failed: %#v", resp) + } + if resp.PrevNode != nil { + t.Fatalf("Create 1 PrevNode failed: %#v", resp) + } + + // This should fail, because the key is already there + resp, err = c.Create(newKey, newValue, 5) + if err == nil { + t.Fatalf("The key %v did exist, so the creation should have failed."+ + "The response was: %#v", resp.Node.Key, resp) + } +} + +func TestCreateInOrder(t *testing.T) { + c := NewClient(nil) + dir := "/queue" + defer func() { + c.DeleteDir(dir) + }() + + var firstKey, secondKey string + + resp, err := c.CreateInOrder(dir, "1", 5) + if err != nil { + t.Fatal(err) + } + + if !(resp.Action == "create" && resp.Node.Value == "1" && resp.Node.TTL == 5) { + t.Fatalf("Create 1 failed: %#v", resp) + } + + firstKey = resp.Node.Key + + resp, err = c.CreateInOrder(dir, "2", 5) + if err != nil { + t.Fatal(err) + } + + if !(resp.Action == "create" && resp.Node.Value == "2" && resp.Node.TTL == 5) { + t.Fatalf("Create 2 failed: %#v", resp) + } + + secondKey = resp.Node.Key + + if firstKey >= secondKey { + t.Fatalf("Expected first key to be greater than second key, but %s is not greater than %s", + firstKey, secondKey) + } +} + +func TestSetDir(t *testing.T) { + c := NewClient(nil) + defer func() { + c.Delete("foo", true) + c.Delete("fooDir", true) + }() + + resp, err := c.CreateDir("fooDir", 5) + if err != nil { + t.Fatal(err) + } + if !(resp.Node.Key == "/fooDir" && resp.Node.Value == "" && resp.Node.TTL == 5) { + t.Fatalf("SetDir 1 failed: %#v", resp) + } + if resp.PrevNode != nil { + t.Fatalf("SetDir 1 PrevNode failed: %#v", resp) + } + + // This should fail because /fooDir already points to a directory + resp, err = c.CreateDir("/fooDir", 5) + if err == nil { + t.Fatalf("fooDir already points to a directory, so SetDir should have failed."+ + "The response was: %#v", resp) + } + + _, err = c.Set("foo", "bar", 5) + if err != nil { + t.Fatal(err) + } + + // This should succeed + // It should replace the key + resp, err = c.SetDir("foo", 5) + if err != nil { + t.Fatal(err) + } + if !(resp.Node.Key == "/foo" && resp.Node.Value == "" && resp.Node.TTL == 5) { + t.Fatalf("SetDir 2 failed: %#v", resp) + } + if !(resp.PrevNode.Key == "/foo" && resp.PrevNode.Value == "bar" && resp.PrevNode.TTL == 5) { + t.Fatalf("SetDir 2 failed: %#v", resp) + } +} + +func TestUpdateDir(t *testing.T) { + c := NewClient(nil) + defer func() { + c.Delete("fooDir", true) + }() + + resp, err := c.CreateDir("fooDir", 5) + if err != nil { + t.Fatal(err) + } + + // This should succeed. + resp, err = c.UpdateDir("fooDir", 5) + if err != nil { + t.Fatal(err) + } + + if !(resp.Action == "update" && resp.Node.Key == "/fooDir" && + resp.Node.Value == "" && resp.Node.TTL == 5) { + t.Fatalf("UpdateDir 1 failed: %#v", resp) + } + if !(resp.PrevNode.Key == "/fooDir" && resp.PrevNode.Dir == true && resp.PrevNode.TTL == 5) { + t.Fatalf("UpdateDir 1 PrevNode failed: %#v", resp) + } + + // This should fail because the key does not exist. + resp, err = c.UpdateDir("nonexistentDir", 5) + if err == nil { + t.Fatalf("The key %v did not exist, so the update should have failed."+ + "The response was: %#v", resp.Node.Key, resp) + } +} + +func TestCreateDir(t *testing.T) { + c := NewClient(nil) + defer func() { + c.Delete("fooDir", true) + }() + + // This should succeed + resp, err := c.CreateDir("fooDir", 5) + if err != nil { + t.Fatal(err) + } + + if !(resp.Action == "create" && resp.Node.Key == "/fooDir" && + resp.Node.Value == "" && resp.Node.TTL == 5) { + t.Fatalf("CreateDir 1 failed: %#v", resp) + } + if resp.PrevNode != nil { + t.Fatalf("CreateDir 1 PrevNode failed: %#v", resp) + } + + // This should fail, because the key is already there + resp, err = c.CreateDir("fooDir", 5) + if err == nil { + t.Fatalf("The key %v did exist, so the creation should have failed."+ + "The response was: %#v", resp.Node.Key, resp) + } +} diff --git a/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/version.go b/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/version.go new file mode 100644 index 000000000..b1e9ed271 --- /dev/null +++ b/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/version.go @@ -0,0 +1,6 @@ +package etcd + +const ( + version = "v2" + packageVersion = "v2.0.0+git" +) diff --git a/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/watch.go b/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/watch.go new file mode 100644 index 000000000..aa8d3df30 --- /dev/null +++ b/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/watch.go @@ -0,0 +1,103 @@ +package etcd + +import ( + "errors" +) + +// Errors introduced by the Watch command. +var ( + ErrWatchStoppedByUser = errors.New("Watch stopped by the user via stop channel") +) + +// If recursive is set to true the watch returns the first change under the given +// prefix since the given index. +// +// If recursive is set to false the watch returns the first change to the given key +// since the given index. +// +// To watch for the latest change, set waitIndex = 0. +// +// If a receiver channel is given, it will be a long-term watch. Watch will block at the +//channel. After someone receives the channel, it will go on to watch that +// prefix. If a stop channel is given, the client can close long-term watch using +// the stop channel. +func (c *Client) Watch(prefix string, waitIndex uint64, recursive bool, + receiver chan *Response, stop chan bool) (*Response, error) { + logger.Debugf("watch %s [%s]", prefix, c.cluster.Leader) + if receiver == nil { + raw, err := c.watchOnce(prefix, waitIndex, recursive, stop) + + if err != nil { + return nil, err + } + + return raw.Unmarshal() + } + defer close(receiver) + + for { + raw, err := c.watchOnce(prefix, waitIndex, recursive, stop) + + if err != nil { + return nil, err + } + + resp, err := raw.Unmarshal() + + if err != nil { + return nil, err + } + + waitIndex = resp.Node.ModifiedIndex + 1 + receiver <- resp + } +} + +func (c *Client) RawWatch(prefix string, waitIndex uint64, recursive bool, + receiver chan *RawResponse, stop chan bool) (*RawResponse, error) { + + logger.Debugf("rawWatch %s [%s]", prefix, c.cluster.Leader) + if receiver == nil { + return c.watchOnce(prefix, waitIndex, recursive, stop) + } + + for { + raw, err := c.watchOnce(prefix, waitIndex, recursive, stop) + + if err != nil { + return nil, err + } + + resp, err := raw.Unmarshal() + + if err != nil { + return nil, err + } + + waitIndex = resp.Node.ModifiedIndex + 1 + receiver <- raw + } +} + +// helper func +// return when there is change under the given prefix +func (c *Client) watchOnce(key string, waitIndex uint64, recursive bool, stop chan bool) (*RawResponse, error) { + + options := Options{ + "wait": true, + } + if waitIndex > 0 { + options["waitIndex"] = waitIndex + } + if recursive { + options["recursive"] = true + } + + resp, err := c.getCancelable(key, options, stop) + + if err == ErrRequestCancelled { + return nil, ErrWatchStoppedByUser + } + + return resp, err +} diff --git a/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/watch_test.go b/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/watch_test.go new file mode 100644 index 000000000..43e1dfeb8 --- /dev/null +++ b/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/watch_test.go @@ -0,0 +1,119 @@ +package etcd + +import ( + "fmt" + "runtime" + "testing" + "time" +) + +func TestWatch(t *testing.T) { + c := NewClient(nil) + defer func() { + c.Delete("watch_foo", true) + }() + + go setHelper("watch_foo", "bar", c) + + resp, err := c.Watch("watch_foo", 0, false, nil, nil) + if err != nil { + t.Fatal(err) + } + if !(resp.Node.Key == "/watch_foo" && resp.Node.Value == "bar") { + t.Fatalf("Watch 1 failed: %#v", resp) + } + + go setHelper("watch_foo", "bar", c) + + resp, err = c.Watch("watch_foo", resp.Node.ModifiedIndex+1, false, nil, nil) + if err != nil { + t.Fatal(err) + } + if !(resp.Node.Key == "/watch_foo" && resp.Node.Value == "bar") { + t.Fatalf("Watch 2 failed: %#v", resp) + } + + routineNum := runtime.NumGoroutine() + + ch := make(chan *Response, 10) + stop := make(chan bool, 1) + + go setLoop("watch_foo", "bar", c) + + go receiver(ch, stop) + + _, err = c.Watch("watch_foo", 0, false, ch, stop) + if err != ErrWatchStoppedByUser { + t.Fatalf("Watch returned a non-user stop error") + } + + if newRoutineNum := runtime.NumGoroutine(); newRoutineNum != routineNum { + t.Fatalf("Routine numbers differ after watch stop: %v, %v", routineNum, newRoutineNum) + } +} + +func TestWatchAll(t *testing.T) { + c := NewClient(nil) + defer func() { + c.Delete("watch_foo", true) + }() + + go setHelper("watch_foo/foo", "bar", c) + + resp, err := c.Watch("watch_foo", 0, true, nil, nil) + if err != nil { + t.Fatal(err) + } + if !(resp.Node.Key == "/watch_foo/foo" && resp.Node.Value == "bar") { + t.Fatalf("WatchAll 1 failed: %#v", resp) + } + + go setHelper("watch_foo/foo", "bar", c) + + resp, err = c.Watch("watch_foo", resp.Node.ModifiedIndex+1, true, nil, nil) + if err != nil { + t.Fatal(err) + } + if !(resp.Node.Key == "/watch_foo/foo" && resp.Node.Value == "bar") { + t.Fatalf("WatchAll 2 failed: %#v", resp) + } + + ch := make(chan *Response, 10) + stop := make(chan bool, 1) + + routineNum := runtime.NumGoroutine() + + go setLoop("watch_foo/foo", "bar", c) + + go receiver(ch, stop) + + _, err = c.Watch("watch_foo", 0, true, ch, stop) + if err != ErrWatchStoppedByUser { + t.Fatalf("Watch returned a non-user stop error") + } + + if newRoutineNum := runtime.NumGoroutine(); newRoutineNum != routineNum { + t.Fatalf("Routine numbers differ after watch stop: %v, %v", routineNum, newRoutineNum) + } +} + +func setHelper(key, value string, c *Client) { + time.Sleep(time.Second) + c.Set(key, value, 100) +} + +func setLoop(key, value string, c *Client) { + time.Sleep(time.Second) + for i := 0; i < 10; i++ { + newValue := fmt.Sprintf("%s_%v", value, i) + c.Set(key, newValue, 100) + time.Sleep(time.Second / 10) + } +} + +func receiver(c chan *Response, stop chan bool) { + for i := 0; i < 10; i++ { + <-c + } + stop <- true +} diff --git a/Godeps/_workspace/src/github.com/docker/docker/pkg/ioutils/fmt.go b/Godeps/_workspace/src/github.com/docker/docker/pkg/ioutils/fmt.go new file mode 100644 index 000000000..801132ff3 --- /dev/null +++ b/Godeps/_workspace/src/github.com/docker/docker/pkg/ioutils/fmt.go @@ -0,0 +1,14 @@ +package ioutils + +import ( + "fmt" + "io" +) + +// FprintfIfNotEmpty prints the string value if it's not empty +func FprintfIfNotEmpty(w io.Writer, format, value string) (int, error) { + if value != "" { + return fmt.Fprintf(w, format, value) + } + return 0, nil +} diff --git a/Godeps/_workspace/src/github.com/docker/docker/pkg/ioutils/fmt_test.go b/Godeps/_workspace/src/github.com/docker/docker/pkg/ioutils/fmt_test.go new file mode 100644 index 000000000..896886329 --- /dev/null +++ b/Godeps/_workspace/src/github.com/docker/docker/pkg/ioutils/fmt_test.go @@ -0,0 +1,17 @@ +package ioutils + +import "testing" + +func TestFprintfIfNotEmpty(t *testing.T) { + wc := NewWriteCounter(&NopWriter{}) + n, _ := FprintfIfNotEmpty(wc, "foo%s", "") + + if wc.Count != 0 || n != 0 { + t.Errorf("Wrong count: %v vs. %v vs. 0", wc.Count, n) + } + + n, _ = FprintfIfNotEmpty(wc, "foo%s", "bar") + if wc.Count != 6 || n != 6 { + t.Errorf("Wrong count: %v vs. %v vs. 6", wc.Count, n) + } +} diff --git a/Godeps/_workspace/src/github.com/docker/docker/pkg/ioutils/multireader.go b/Godeps/_workspace/src/github.com/docker/docker/pkg/ioutils/multireader.go new file mode 100644 index 000000000..0d2d76b47 --- /dev/null +++ b/Godeps/_workspace/src/github.com/docker/docker/pkg/ioutils/multireader.go @@ -0,0 +1,226 @@ +package ioutils + +import ( + "bytes" + "fmt" + "io" + "os" +) + +type pos struct { + idx int + offset int64 +} + +type multiReadSeeker struct { + readers []io.ReadSeeker + pos *pos + posIdx map[io.ReadSeeker]int +} + +func (r *multiReadSeeker) Seek(offset int64, whence int) (int64, error) { + var tmpOffset int64 + switch whence { + case os.SEEK_SET: + for i, rdr := range r.readers { + // get size of the current reader + s, err := rdr.Seek(0, os.SEEK_END) + if err != nil { + return -1, err + } + + if offset > tmpOffset+s { + if i == len(r.readers)-1 { + rdrOffset := s + (offset - tmpOffset) + if _, err := rdr.Seek(rdrOffset, os.SEEK_SET); err != nil { + return -1, err + } + r.pos = &pos{i, rdrOffset} + return offset, nil + } + + tmpOffset += s + continue + } + + rdrOffset := offset - tmpOffset + idx := i + + rdr.Seek(rdrOffset, os.SEEK_SET) + // make sure all following readers are at 0 + for _, rdr := range r.readers[i+1:] { + rdr.Seek(0, os.SEEK_SET) + } + + if rdrOffset == s && i != len(r.readers)-1 { + idx++ + rdrOffset = 0 + } + r.pos = &pos{idx, rdrOffset} + return offset, nil + } + case os.SEEK_END: + for _, rdr := range r.readers { + s, err := rdr.Seek(0, os.SEEK_END) + if err != nil { + return -1, err + } + tmpOffset += s + } + r.Seek(tmpOffset+offset, os.SEEK_SET) + return tmpOffset + offset, nil + case os.SEEK_CUR: + if r.pos == nil { + return r.Seek(offset, os.SEEK_SET) + } + // Just return the current offset + if offset == 0 { + return r.getCurOffset() + } + + curOffset, err := r.getCurOffset() + if err != nil { + return -1, err + } + rdr, rdrOffset, err := r.getReaderForOffset(curOffset + offset) + if err != nil { + return -1, err + } + + r.pos = &pos{r.posIdx[rdr], rdrOffset} + return curOffset + offset, nil + default: + return -1, fmt.Errorf("Invalid whence: %d", whence) + } + + return -1, fmt.Errorf("Error seeking for whence: %d, offset: %d", whence, offset) +} + +func (r *multiReadSeeker) getReaderForOffset(offset int64) (io.ReadSeeker, int64, error) { + var rdr io.ReadSeeker + var rdrOffset int64 + + for i, rdr := range r.readers { + offsetTo, err := r.getOffsetToReader(rdr) + if err != nil { + return nil, -1, err + } + if offsetTo > offset { + rdr = r.readers[i-1] + rdrOffset = offsetTo - offset + break + } + + if rdr == r.readers[len(r.readers)-1] { + rdrOffset = offsetTo + offset + break + } + } + + return rdr, rdrOffset, nil +} + +func (r *multiReadSeeker) getCurOffset() (int64, error) { + var totalSize int64 + for _, rdr := range r.readers[:r.pos.idx+1] { + if r.posIdx[rdr] == r.pos.idx { + totalSize += r.pos.offset + break + } + + size, err := getReadSeekerSize(rdr) + if err != nil { + return -1, fmt.Errorf("error getting seeker size: %v", err) + } + totalSize += size + } + return totalSize, nil +} + +func (r *multiReadSeeker) getOffsetToReader(rdr io.ReadSeeker) (int64, error) { + var offset int64 + for _, r := range r.readers { + if r == rdr { + break + } + + size, err := getReadSeekerSize(rdr) + if err != nil { + return -1, err + } + offset += size + } + return offset, nil +} + +func (r *multiReadSeeker) Read(b []byte) (int, error) { + if r.pos == nil { + r.pos = &pos{0, 0} + } + + bCap := int64(cap(b)) + buf := bytes.NewBuffer(nil) + var rdr io.ReadSeeker + + for _, rdr = range r.readers[r.pos.idx:] { + readBytes, err := io.CopyN(buf, rdr, bCap) + if err != nil && err != io.EOF { + return -1, err + } + bCap -= readBytes + + if bCap == 0 { + break + } + } + + rdrPos, err := rdr.Seek(0, os.SEEK_CUR) + if err != nil { + return -1, err + } + r.pos = &pos{r.posIdx[rdr], rdrPos} + return buf.Read(b) +} + +func getReadSeekerSize(rdr io.ReadSeeker) (int64, error) { + // save the current position + pos, err := rdr.Seek(0, os.SEEK_CUR) + if err != nil { + return -1, err + } + + // get the size + size, err := rdr.Seek(0, os.SEEK_END) + if err != nil { + return -1, err + } + + // reset the position + if _, err := rdr.Seek(pos, os.SEEK_SET); err != nil { + return -1, err + } + return size, nil +} + +// MultiReadSeeker returns a ReadSeeker that's the logical concatenation of the provided +// input readseekers. After calling this method the initial position is set to the +// beginning of the first ReadSeeker. At the end of a ReadSeeker, Read always advances +// to the beginning of the next ReadSeeker and returns EOF at the end of the last ReadSeeker. +// Seek can be used over the sum of lengths of all readseekers. +// +// When a MultiReadSeeker is used, no Read and Seek operations should be made on +// its ReadSeeker components. Also, users should make no assumption on the state +// of individual readseekers while the MultiReadSeeker is used. +func MultiReadSeeker(readers ...io.ReadSeeker) io.ReadSeeker { + if len(readers) == 1 { + return readers[0] + } + idx := make(map[io.ReadSeeker]int) + for i, rdr := range readers { + idx[rdr] = i + } + return &multiReadSeeker{ + readers: readers, + posIdx: idx, + } +} diff --git a/Godeps/_workspace/src/github.com/docker/docker/pkg/ioutils/multireader_test.go b/Godeps/_workspace/src/github.com/docker/docker/pkg/ioutils/multireader_test.go new file mode 100644 index 000000000..de495b56d --- /dev/null +++ b/Godeps/_workspace/src/github.com/docker/docker/pkg/ioutils/multireader_test.go @@ -0,0 +1,149 @@ +package ioutils + +import ( + "bytes" + "fmt" + "io" + "io/ioutil" + "os" + "strings" + "testing" +) + +func TestMultiReadSeekerReadAll(t *testing.T) { + str := "hello world" + s1 := strings.NewReader(str + " 1") + s2 := strings.NewReader(str + " 2") + s3 := strings.NewReader(str + " 3") + mr := MultiReadSeeker(s1, s2, s3) + + expectedSize := int64(s1.Len() + s2.Len() + s3.Len()) + + b, err := ioutil.ReadAll(mr) + if err != nil { + t.Fatal(err) + } + + expected := "hello world 1hello world 2hello world 3" + if string(b) != expected { + t.Fatalf("ReadAll failed, got: %q, expected %q", string(b), expected) + } + + size, err := mr.Seek(0, os.SEEK_END) + if err != nil { + t.Fatal(err) + } + if size != expectedSize { + t.Fatalf("reader size does not match, got %d, expected %d", size, expectedSize) + } + + // Reset the position and read again + pos, err := mr.Seek(0, os.SEEK_SET) + if err != nil { + t.Fatal(err) + } + if pos != 0 { + t.Fatalf("expected position to be set to 0, got %d", pos) + } + + b, err = ioutil.ReadAll(mr) + if err != nil { + t.Fatal(err) + } + + if string(b) != expected { + t.Fatalf("ReadAll failed, got: %q, expected %q", string(b), expected) + } +} + +func TestMultiReadSeekerReadEach(t *testing.T) { + str := "hello world" + s1 := strings.NewReader(str + " 1") + s2 := strings.NewReader(str + " 2") + s3 := strings.NewReader(str + " 3") + mr := MultiReadSeeker(s1, s2, s3) + + var totalBytes int64 + for i, s := range []*strings.Reader{s1, s2, s3} { + sLen := int64(s.Len()) + buf := make([]byte, s.Len()) + expected := []byte(fmt.Sprintf("%s %d", str, i+1)) + + if _, err := mr.Read(buf); err != nil && err != io.EOF { + t.Fatal(err) + } + + if !bytes.Equal(buf, expected) { + t.Fatalf("expected %q to be %q", string(buf), string(expected)) + } + + pos, err := mr.Seek(0, os.SEEK_CUR) + if err != nil { + t.Fatalf("iteration: %d, error: %v", i+1, err) + } + + // check that the total bytes read is the current position of the seeker + totalBytes += sLen + if pos != totalBytes { + t.Fatalf("expected current position to be: %d, got: %d, iteration: %d", totalBytes, pos, i+1) + } + + // This tests not only that SEEK_SET and SEEK_CUR give the same values, but that the next iteration is in the expected position as well + newPos, err := mr.Seek(pos, os.SEEK_SET) + if err != nil { + t.Fatal(err) + } + if newPos != pos { + t.Fatalf("expected to get same position when calling SEEK_SET with value from SEEK_CUR, cur: %d, set: %d", pos, newPos) + } + } +} + +func TestMultiReadSeekerReadSpanningChunks(t *testing.T) { + str := "hello world" + s1 := strings.NewReader(str + " 1") + s2 := strings.NewReader(str + " 2") + s3 := strings.NewReader(str + " 3") + mr := MultiReadSeeker(s1, s2, s3) + + buf := make([]byte, s1.Len()+3) + _, err := mr.Read(buf) + if err != nil { + t.Fatal(err) + } + + // expected is the contents of s1 + 3 bytes from s2, ie, the `hel` at the end of this string + expected := "hello world 1hel" + if string(buf) != expected { + t.Fatalf("expected %s to be %s", string(buf), expected) + } +} + +func TestMultiReadSeekerNegativeSeek(t *testing.T) { + str := "hello world" + s1 := strings.NewReader(str + " 1") + s2 := strings.NewReader(str + " 2") + s3 := strings.NewReader(str + " 3") + mr := MultiReadSeeker(s1, s2, s3) + + s1Len := s1.Len() + s2Len := s2.Len() + s3Len := s3.Len() + + s, err := mr.Seek(int64(-1*s3.Len()), os.SEEK_END) + if err != nil { + t.Fatal(err) + } + if s != int64(s1Len+s2Len) { + t.Fatalf("expected %d to be %d", s, s1.Len()+s2.Len()) + } + + buf := make([]byte, s3Len) + if _, err := mr.Read(buf); err != nil && err != io.EOF { + t.Fatal(err) + } + expected := fmt.Sprintf("%s %d", str, 3) + if string(buf) != fmt.Sprintf("%s %d", str, 3) { + t.Fatalf("expected %q to be %q", string(buf), expected) + } +} diff --git a/Godeps/_workspace/src/github.com/docker/docker/pkg/ioutils/readers.go b/Godeps/_workspace/src/github.com/docker/docker/pkg/ioutils/readers.go new file mode 100644 index 000000000..10ebcb5ed --- /dev/null +++ b/Godeps/_workspace/src/github.com/docker/docker/pkg/ioutils/readers.go @@ -0,0 +1,262 @@ +package ioutils + +import ( + "bytes" + "crypto/sha256" + "encoding/hex" + "io" + "math/rand" + "sync" + "time" + + "github.com/docker/docker/pkg/random" +) + +var rndSrc = random.NewSource() + +type readCloserWrapper struct { + io.Reader + closer func() error +} + +func (r *readCloserWrapper) Close() error { + return r.closer() +} + +// NewReadCloserWrapper returns a new io.ReadCloser. +func NewReadCloserWrapper(r io.Reader, closer func() error) io.ReadCloser { + return &readCloserWrapper{ + Reader: r, + closer: closer, + } +} + +type readerErrWrapper struct { + reader io.Reader + closer func() +} + +func (r *readerErrWrapper) Read(p []byte) (int, error) { + n, err := r.reader.Read(p) + if err != nil { + r.closer() + } + return n, err +} + +// NewReaderErrWrapper returns a new io.Reader. +func NewReaderErrWrapper(r io.Reader, closer func()) io.Reader { + return &readerErrWrapper{ + reader: r, + closer: closer, + } +} + +// bufReader allows the underlying reader to continue to produce +// output by pre-emptively reading from the wrapped reader. +// This is achieved by buffering this data in bufReader's +// expanding buffer. +type bufReader struct { + sync.Mutex + buf *bytes.Buffer + reader io.Reader + err error + wait sync.Cond + drainBuf []byte + reuseBuf []byte + maxReuse int64 + resetTimeout time.Duration + bufLenResetThreshold int64 + maxReadDataReset int64 +} + +// NewBufReader returns a new bufReader. +func NewBufReader(r io.Reader) io.ReadCloser { + timeout := rand.New(rndSrc).Intn(120) + 180 + + reader := &bufReader{ + buf: &bytes.Buffer{}, + drainBuf: make([]byte, 1024), + reuseBuf: make([]byte, 4096), + maxReuse: 1000, + resetTimeout: time.Duration(timeout) * time.Second, + bufLenResetThreshold: 100 * 1024, + maxReadDataReset: 10 * 1024 * 1024, + reader: r, + } + reader.wait.L = &reader.Mutex + go reader.drain() + return reader +} + +// NewBufReaderWithDrainbufAndBuffer returns a BufReader with drainBuffer and buffer. +func NewBufReaderWithDrainbufAndBuffer(r io.Reader, drainBuffer []byte, buffer *bytes.Buffer) io.ReadCloser { + reader := &bufReader{ + buf: buffer, + drainBuf: drainBuffer, + reader: r, + } + reader.wait.L = &reader.Mutex + go reader.drain() + return reader +} + +func (r *bufReader) drain() { + var ( + duration time.Duration + lastReset time.Time + now time.Time + reset bool + bufLen int64 + dataSinceReset int64 + maxBufLen int64 + reuseBufLen int64 + reuseCount int64 + ) + reuseBufLen = int64(len(r.reuseBuf)) + lastReset = time.Now() + for { + n, err := r.reader.Read(r.drainBuf) + dataSinceReset += int64(n) + r.Lock() + bufLen = int64(r.buf.Len()) + if bufLen > maxBufLen { + maxBufLen = bufLen + } + + // Avoid unbounded growth of the buffer over time. + // This has been discovered to be the only non-intrusive + // solution to the unbounded growth of the buffer. + // Alternative solutions such as compression, multiple + // buffers, channels and other similar pieces of code + // were reducing throughput, overall Docker performance + // or simply crashed Docker. + // This solution releases the buffer when specific + // conditions are met to avoid the continuous resizing + // of the buffer for long lived containers. + // + // Move data to the front of the buffer if it's + // smaller than what reuseBuf can store + if bufLen > 0 && reuseBufLen >= bufLen { + n, _ := r.buf.Read(r.reuseBuf) + r.buf.Write(r.reuseBuf[0:n]) + // Take action if the buffer has been reused too many + // times and if there's data in the buffer. + // The timeout is also used as means to avoid doing + // these operations more often or less often than + // required. + // The various conditions try to detect heavy activity + // in the buffer which might be indicators of heavy + // growth of the buffer. + } else if reuseCount >= r.maxReuse && bufLen > 0 { + now = time.Now() + duration = now.Sub(lastReset) + timeoutReached := duration >= r.resetTimeout + + // The timeout has been reached and the + // buffered data couldn't be moved to the front + // of the buffer, so the buffer gets reset. + if timeoutReached && bufLen > reuseBufLen { + reset = true + } + // The amount of buffered data is too high now, + // reset the buffer. + if timeoutReached && maxBufLen >= r.bufLenResetThreshold { + reset = true + } + // Reset the buffer if a certain amount of + // data has gone through the buffer since the + // last reset. + if timeoutReached && dataSinceReset >= r.maxReadDataReset { + reset = true + } + // The buffered data is moved to a fresh buffer, + // swap the old buffer with the new one and + // reset all counters. + if reset { + newbuf := &bytes.Buffer{} + newbuf.ReadFrom(r.buf) + r.buf = newbuf + lastReset = now + reset = false + dataSinceReset = 0 + maxBufLen = 0 + reuseCount = 0 + } + } + if err != nil { + r.err = err + } else { + r.buf.Write(r.drainBuf[0:n]) + } + reuseCount++ + r.wait.Signal() + r.Unlock() + callSchedulerIfNecessary() + if err != nil { + break + } + } +} + +func (r *bufReader) Read(p []byte) (n int, err error) { + r.Lock() + defer r.Unlock() + for { + n, err = r.buf.Read(p) + if n > 0 { + return n, err + } + if r.err != nil { + return 0, r.err + } + r.wait.Wait() + } +} + +// Close closes the bufReader +func (r *bufReader) Close() error { + closer, ok := r.reader.(io.ReadCloser) + if !ok { + return nil + } + return closer.Close() +} + +// HashData returns the sha256 sum of src. +func HashData(src io.Reader) (string, error) { + h := sha256.New() + if _, err := io.Copy(h, src); err != nil { + return "", err + } + return "sha256:" + hex.EncodeToString(h.Sum(nil)), nil +} + +// OnEOFReader wraps a io.ReadCloser and a function +// the fuction will run at the end of file or close the file. +type OnEOFReader struct { + Rc io.ReadCloser + Fn func() +} + +func (r *OnEOFReader) Read(p []byte) (n int, err error) { + n, err = r.Rc.Read(p) + if err == io.EOF { + r.runFunc() + } + return +} + +// Close closes the file and run the function. +func (r *OnEOFReader) Close() error { + err := r.Rc.Close() + r.runFunc() + return err +} + +func (r *OnEOFReader) runFunc() { + if fn := r.Fn; fn != nil { + fn() + r.Fn = nil + } +} diff --git a/Godeps/_workspace/src/github.com/docker/docker/pkg/ioutils/readers_test.go b/Godeps/_workspace/src/github.com/docker/docker/pkg/ioutils/readers_test.go new file mode 100644 index 000000000..0a39b6ec6 --- /dev/null +++ b/Godeps/_workspace/src/github.com/docker/docker/pkg/ioutils/readers_test.go @@ -0,0 +1,216 @@ +package ioutils + +import ( + "bytes" + "fmt" + "io" + "io/ioutil" + "strings" + "testing" +) + +// Implement io.Reader +type errorReader struct{} + +func (r *errorReader) Read(p []byte) (int, error) { + return 0, fmt.Errorf("Error reader always fail.") +} + +func TestReadCloserWrapperClose(t *testing.T) { + reader := strings.NewReader("A string reader") + wrapper := NewReadCloserWrapper(reader, func() error { + return fmt.Errorf("This will be called when closing") + }) + err := wrapper.Close() + if err == nil || !strings.Contains(err.Error(), "This will be called when closing") { + t.Fatalf("readCloserWrapper should have call the anonymous func and thus, fail.") + } +} + +func TestReaderErrWrapperReadOnError(t *testing.T) { + called := false + reader := &errorReader{} + wrapper := NewReaderErrWrapper(reader, func() { + called = true + }) + _, err := wrapper.Read([]byte{}) + if err == nil || !strings.Contains(err.Error(), "Error reader always fail.") { + t.Fatalf("readErrWrapper should returned an error") + } + if !called { + t.Fatalf("readErrWrapper should have call the anonymous function on failure") + } +} + +func TestReaderErrWrapperRead(t *testing.T) { + reader := strings.NewReader("a string reader.") + wrapper := NewReaderErrWrapper(reader, func() { + t.Fatalf("readErrWrapper should not have called the anonymous function") + }) + // Read 20 byte (should be ok with the string above) + num, err := wrapper.Read(make([]byte, 20)) + if err != nil { + t.Fatal(err) + } + if num != 16 { + t.Fatalf("readerErrWrapper should have read 16 byte, but read %d", num) + } +} + +func TestNewBufReaderWithDrainbufAndBuffer(t *testing.T) { + reader, writer := io.Pipe() + + drainBuffer := make([]byte, 1024) + buffer := bytes.Buffer{} + bufreader := NewBufReaderWithDrainbufAndBuffer(reader, drainBuffer, &buffer) + + // Write everything down to a Pipe + // Usually, a pipe should block but because of the buffered reader, + // the writes will go through + done := make(chan bool) + go func() { + writer.Write([]byte("hello world")) + writer.Close() + done <- true + }() + + // Drain the reader *after* everything has been written, just to verify + // it is indeed buffering + <-done + + output, err := ioutil.ReadAll(bufreader) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(output, []byte("hello world")) { + t.Error(string(output)) + } +} + +func TestBufReader(t *testing.T) { + reader, writer := io.Pipe() + bufreader := NewBufReader(reader) + + // Write everything down to a Pipe + // Usually, a pipe should block but because of the buffered reader, + // the writes will go through + done := make(chan bool) + go func() { + writer.Write([]byte("hello world")) + writer.Close() + done <- true + }() + + // Drain the reader *after* everything has been written, just to verify + // it is indeed buffering + <-done + output, err := ioutil.ReadAll(bufreader) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(output, []byte("hello world")) { + t.Error(string(output)) + } +} + +func TestBufReaderCloseWithNonReaderCloser(t *testing.T) { + reader := strings.NewReader("buffer") + bufreader := NewBufReader(reader) + + if err := bufreader.Close(); err != nil { + t.Fatal(err) + } + +} + +// implements io.ReadCloser +type simpleReaderCloser struct{} + +func (r *simpleReaderCloser) Read(p []byte) (n int, err error) { + return 0, nil +} + +func (r *simpleReaderCloser) Close() error { + return nil +} + +func TestBufReaderCloseWithReaderCloser(t *testing.T) { + reader := &simpleReaderCloser{} + bufreader := NewBufReader(reader) + + err := bufreader.Close() + if err != nil { + t.Fatal(err) + } + +} + +func TestHashData(t *testing.T) { + reader := strings.NewReader("hash-me") + actual, err := HashData(reader) + if err != nil { + t.Fatal(err) + } + expected := "sha256:4d11186aed035cc624d553e10db358492c84a7cd6b9670d92123c144930450aa" + if actual != expected { + t.Fatalf("Expecting %s, got %s", expected, actual) + } +} + +type repeatedReader struct { + readCount int + maxReads int + data []byte +} + +func newRepeatedReader(max int, data []byte) *repeatedReader { + return &repeatedReader{0, max, data} +} + +func (r *repeatedReader) Read(p []byte) (int, error) { + if r.readCount >= r.maxReads { + return 0, io.EOF + } + r.readCount++ + n := copy(p, r.data) + return n, nil +} + +func testWithData(data []byte, reads int) { + reader := newRepeatedReader(reads, data) + bufReader := NewBufReader(reader) + io.Copy(ioutil.Discard, bufReader) +} + +func Benchmark1M10BytesReads(b *testing.B) { + reads := 1000000 + readSize := int64(10) + data := make([]byte, readSize) + b.SetBytes(readSize * int64(reads)) + b.ResetTimer() + for i := 0; i < b.N; i++ { + testWithData(data, reads) + } +} + +func Benchmark1M1024BytesReads(b *testing.B) { + reads := 1000000 + readSize := int64(1024) + data := make([]byte, readSize) + b.SetBytes(readSize * int64(reads)) + b.ResetTimer() + for i := 0; i < b.N; i++ { + testWithData(data, reads) + } +} + +func Benchmark10k32KBytesReads(b *testing.B) { + reads := 10000 + readSize := int64(32 * 1024) + data := make([]byte, readSize) + b.SetBytes(readSize * int64(reads)) + b.ResetTimer() + for i := 0; i < b.N; i++ { + testWithData(data, reads) + } +} diff --git a/Godeps/_workspace/src/github.com/docker/docker/pkg/ioutils/scheduler.go b/Godeps/_workspace/src/github.com/docker/docker/pkg/ioutils/scheduler.go new file mode 100644 index 000000000..3c88f29e3 --- /dev/null +++ b/Godeps/_workspace/src/github.com/docker/docker/pkg/ioutils/scheduler.go @@ -0,0 +1,6 @@ +// +build !gccgo + +package ioutils + +func callSchedulerIfNecessary() { +} diff --git a/Godeps/_workspace/src/github.com/docker/docker/pkg/ioutils/scheduler_gccgo.go b/Godeps/_workspace/src/github.com/docker/docker/pkg/ioutils/scheduler_gccgo.go new file mode 100644 index 000000000..c11d02b94 --- /dev/null +++ b/Godeps/_workspace/src/github.com/docker/docker/pkg/ioutils/scheduler_gccgo.go @@ -0,0 +1,13 @@ +// +build gccgo + +package ioutils + +import ( + "runtime" +) + +func callSchedulerIfNecessary() { + //allow or force Go scheduler to switch context, without explicitly + //forcing this will make it hang when using gccgo implementation + runtime.Gosched() +} diff --git a/Godeps/_workspace/src/github.com/docker/docker/pkg/ioutils/writeflusher.go b/Godeps/_workspace/src/github.com/docker/docker/pkg/ioutils/writeflusher.go new file mode 100644 index 000000000..cedb9a0dd --- /dev/null +++ b/Godeps/_workspace/src/github.com/docker/docker/pkg/ioutils/writeflusher.go @@ -0,0 +1,51 @@ +package ioutils + +import ( + "io" + "net/http" + "sync" +) + +// WriteFlusher wraps the Write and Flush operation. +type WriteFlusher struct { + sync.Mutex + w io.Writer + flusher http.Flusher + flushed bool +} + +func (wf *WriteFlusher) Write(b []byte) (n int, err error) { + wf.Lock() + defer wf.Unlock() + n, err = wf.w.Write(b) + wf.flushed = true + wf.flusher.Flush() + return n, err +} + +// Flush the stream immediately. +func (wf *WriteFlusher) Flush() { + wf.Lock() + defer wf.Unlock() + wf.flushed = true + wf.flusher.Flush() +} + +// Flushed returns the state of flushed. +// If it's flushed, return true, or else it return false. +func (wf *WriteFlusher) Flushed() bool { + wf.Lock() + defer wf.Unlock() + return wf.flushed +} + +// NewWriteFlusher returns a new WriteFlusher. +func NewWriteFlusher(w io.Writer) *WriteFlusher { + var flusher http.Flusher + if f, ok := w.(http.Flusher); ok { + flusher = f + } else { + flusher = &NopFlusher{} + } + return &WriteFlusher{w: w, flusher: flusher} +} diff --git a/Godeps/_workspace/src/github.com/docker/docker/pkg/ioutils/writers.go b/Godeps/_workspace/src/github.com/docker/docker/pkg/ioutils/writers.go new file mode 100644 index 000000000..7a3249f3a --- /dev/null +++ b/Godeps/_workspace/src/github.com/docker/docker/pkg/ioutils/writers.go @@ -0,0 +1,66 @@ +package ioutils + +import "io" + +// NopWriter represents a type which write operation is nop. +type NopWriter struct{} + +func (*NopWriter) Write(buf []byte) (int, error) { + return len(buf), nil +} + +type nopWriteCloser struct { + io.Writer +} + +func (w *nopWriteCloser) Close() error { return nil } + +// NopWriteCloser returns a nopWriteCloser. +func NopWriteCloser(w io.Writer) io.WriteCloser { + return &nopWriteCloser{w} +} + +// NopFlusher represents a type which flush opetatin is nop. +type NopFlusher struct{} + +// Flush is a nop operation. +func (f *NopFlusher) Flush() {} + +type writeCloserWrapper struct { + io.Writer + closer func() error +} + +func (r *writeCloserWrapper) Close() error { + return r.closer() +} + +// NewWriteCloserWrapper returns a new io.WriteCloser. +func NewWriteCloserWrapper(r io.Writer, closer func() error) io.WriteCloser { + return &writeCloserWrapper{ + Writer: r, + closer: closer, + } +} + +// WriteCounter wraps a concrete io.Writer and hold a count of the number +// of bytes written to the writer during a "session". +// This can be convenient when write return is masked +// (e.g., json.Encoder.Encode()) +type WriteCounter struct { + Count int64 + Writer io.Writer +} + +// NewWriteCounter returns a new WriteCounter. +func NewWriteCounter(w io.Writer) *WriteCounter { + return &WriteCounter{ + Writer: w, + } +} + +func (wc *WriteCounter) Write(p []byte) (count int, err error) { + count, err = wc.Writer.Write(p) + wc.Count += int64(count) + return +} diff --git a/Godeps/_workspace/src/github.com/docker/docker/pkg/ioutils/writers_test.go b/Godeps/_workspace/src/github.com/docker/docker/pkg/ioutils/writers_test.go new file mode 100644 index 000000000..564b1cd4f --- /dev/null +++ b/Godeps/_workspace/src/github.com/docker/docker/pkg/ioutils/writers_test.go @@ -0,0 +1,65 @@ +package ioutils + +import ( + "bytes" + "strings" + "testing" +) + +func TestWriteCloserWrapperClose(t *testing.T) { + called := false + writer := bytes.NewBuffer([]byte{}) + wrapper := NewWriteCloserWrapper(writer, func() error { + called = true + return nil + }) + if err := wrapper.Close(); err != nil { + t.Fatal(err) + } + if !called { + t.Fatalf("writeCloserWrapper should have call the anonymous function.") + } +} + +func TestNopWriteCloser(t *testing.T) { + writer := bytes.NewBuffer([]byte{}) + wrapper := NopWriteCloser(writer) + if err := wrapper.Close(); err != nil { + t.Fatal("NopWriteCloser always return nil on Close.") + } + +} + +func TestNopWriter(t *testing.T) { + nw := &NopWriter{} + l, err := nw.Write([]byte{'c'}) + if err != nil { + t.Fatal(err) + } + if l != 1 { + t.Fatalf("Expected 1 got %d", l) + } +} + +func TestWriteCounter(t *testing.T) { + dummy1 := "This is a dummy string." + dummy2 := "This is another dummy string." + totalLength := int64(len(dummy1) + len(dummy2)) + + reader1 := strings.NewReader(dummy1) + reader2 := strings.NewReader(dummy2) + + var buffer bytes.Buffer + wc := NewWriteCounter(&buffer) + + reader1.WriteTo(wc) + reader2.WriteTo(wc) + + if wc.Count != totalLength { + t.Errorf("Wrong count: %d vs. %d", wc.Count, totalLength) + } + + if buffer.String() != dummy1+dummy2 { + t.Error("Wrong message written") + } +} diff --git a/Godeps/_workspace/src/github.com/docker/docker/pkg/parsers/kernel/kernel.go b/Godeps/_workspace/src/github.com/docker/docker/pkg/parsers/kernel/kernel.go new file mode 100644 index 000000000..a21ba137e --- /dev/null +++ b/Godeps/_workspace/src/github.com/docker/docker/pkg/parsers/kernel/kernel.go @@ -0,0 +1,100 @@ +// +build !windows + +// Package kernel provides helper function to get, parse and compare kernel +// versions for different platforms. +package kernel + +import ( + "bytes" + "errors" + "fmt" +) + +// VersionInfo holds information about the kernel. +type VersionInfo struct { + Kernel int // Version of the kernel (e.g. 4.1.2-generic -> 4) + Major int // Major part of the kernel version (e.g. 4.1.2-generic -> 1) + Minor int // Minor part of the kernel version (e.g. 4.1.2-generic -> 2) + Flavor string // Flavor of the kernel version (e.g. 4.1.2-generic -> generic) +} + +func (k *VersionInfo) String() string { + return fmt.Sprintf("%d.%d.%d%s", k.Kernel, k.Major, k.Minor, k.Flavor) +} + +// CompareKernelVersion compares two kernel.VersionInfo structs. +// Returns -1 if a < b, 0 if a == b, 1 it a > b +func CompareKernelVersion(a, b VersionInfo) int { + if a.Kernel < b.Kernel { + return -1 + } else if a.Kernel > b.Kernel { + return 1 + } + + if a.Major < b.Major { + return -1 + } else if a.Major > b.Major { + return 1 + } + + if a.Minor < b.Minor { + return -1 + } else if a.Minor > b.Minor { + return 1 + } + + return 0 +} + +// GetKernelVersion gets the current kernel version. +func GetKernelVersion() (*VersionInfo, error) { + var ( + err error + ) + + uts, err := uname() + if err != nil { + return nil, err + } + + release := make([]byte, len(uts.Release)) + + i := 0 + for _, c := range uts.Release { + release[i] = byte(c) + i++ + } + + // Remove the \x00 from the release for Atoi to parse correctly + release = release[:bytes.IndexByte(release, 0)] + + return ParseRelease(string(release)) +} + +// ParseRelease parses a string and creates a VersionInfo based on it. +func ParseRelease(release string) (*VersionInfo, error) { + var ( + kernel, major, minor, parsed int + flavor, partial string + ) + + // Ignore error from Sscanf to allow an empty flavor. Instead, just + // make sure we got all the version numbers. + parsed, _ = fmt.Sscanf(release, "%d.%d%s", &kernel, &major, &partial) + if parsed < 2 { + return nil, errors.New("Can't parse kernel version " + release) + } + + // sometimes we have 3.12.25-gentoo, but sometimes we just have 3.12-1-amd64 + parsed, _ = fmt.Sscanf(partial, ".%d%s", &minor, &flavor) + if parsed < 1 { + flavor = partial + } + + return &VersionInfo{ + Kernel: kernel, + Major: major, + Minor: minor, + Flavor: flavor, + }, nil +} diff --git a/Godeps/_workspace/src/github.com/docker/docker/pkg/parsers/kernel/kernel_test.go b/Godeps/_workspace/src/github.com/docker/docker/pkg/parsers/kernel/kernel_test.go new file mode 100644 index 000000000..6a2c24680 --- /dev/null +++ b/Godeps/_workspace/src/github.com/docker/docker/pkg/parsers/kernel/kernel_test.go @@ -0,0 +1,92 @@ +package kernel + +import ( + "fmt" + "testing" +) + +func assertParseRelease(t *testing.T, release string, b *VersionInfo, result int) { + var ( + a *VersionInfo + ) + a, _ = ParseRelease(release) + + if r := CompareKernelVersion(*a, *b); r != result { + t.Fatalf("Unexpected kernel version comparison result for (%v,%v). Found %d, expected %d", release, b, r, result) + } + if a.Flavor != b.Flavor { + t.Fatalf("Unexpected parsed kernel flavor. Found %s, expected %s", a.Flavor, b.Flavor) + } +} + +func TestParseRelease(t *testing.T) { + assertParseRelease(t, "3.8.0", &VersionInfo{Kernel: 3, Major: 8, Minor: 0}, 0) + assertParseRelease(t, "3.4.54.longterm-1", &VersionInfo{Kernel: 3, Major: 4, Minor: 54, Flavor: ".longterm-1"}, 0) + assertParseRelease(t, "3.4.54.longterm-1", &VersionInfo{Kernel: 3, Major: 4, Minor: 54, Flavor: ".longterm-1"}, 0) + assertParseRelease(t, "3.8.0-19-generic", &VersionInfo{Kernel: 3, Major: 8, Minor: 0, Flavor: "-19-generic"}, 0) + assertParseRelease(t, "3.12.8tag", &VersionInfo{Kernel: 3, Major: 12, Minor: 8, Flavor: "tag"}, 0) + assertParseRelease(t, "3.12-1-amd64", &VersionInfo{Kernel: 3, Major: 12, Minor: 0, Flavor: "-1-amd64"}, 0) + assertParseRelease(t, "3.8.0", &VersionInfo{Kernel: 4, Major: 8, Minor: 0}, -1) + // Errors + invalids := []string{ + "3", + "a", + "a.a", + "a.a.a-a", + } + for _, invalid := range invalids { + expectedMessage := fmt.Sprintf("Can't parse kernel version %v", invalid) + if _, err := ParseRelease(invalid); err == nil || err.Error() != expectedMessage { + + } + } +} + +func assertKernelVersion(t *testing.T, a, b VersionInfo, result int) { + if r := CompareKernelVersion(a, b); r != result { + t.Fatalf("Unexpected kernel version comparison result. Found %d, expected %d", r, result) + } +} + +func TestCompareKernelVersion(t *testing.T) { + assertKernelVersion(t, + VersionInfo{Kernel: 3, Major: 8, Minor: 0}, + VersionInfo{Kernel: 3, Major: 8, Minor: 0}, + 0) + assertKernelVersion(t, + VersionInfo{Kernel: 2, Major: 6, Minor: 0}, + VersionInfo{Kernel: 3, Major: 8, Minor: 0}, + -1) + assertKernelVersion(t, + VersionInfo{Kernel: 3, Major: 8, Minor: 0}, + VersionInfo{Kernel: 2, Major: 6, Minor: 0}, + 1) + assertKernelVersion(t, + VersionInfo{Kernel: 3, Major: 8, Minor: 0}, + VersionInfo{Kernel: 3, Major: 8, Minor: 0}, + 0) + assertKernelVersion(t, + VersionInfo{Kernel: 3, Major: 8, Minor: 5}, + VersionInfo{Kernel: 3, Major: 8, Minor: 0}, + 1) + assertKernelVersion(t, + VersionInfo{Kernel: 3, Major: 0, Minor: 20}, + VersionInfo{Kernel: 3, Major: 8, Minor: 0}, + -1) + assertKernelVersion(t, + VersionInfo{Kernel: 3, Major: 7, Minor: 20}, + VersionInfo{Kernel: 3, Major: 8, Minor: 0}, + -1) + assertKernelVersion(t, + VersionInfo{Kernel: 3, Major: 8, Minor: 20}, + VersionInfo{Kernel: 3, Major: 7, Minor: 0}, + 1) + assertKernelVersion(t, + VersionInfo{Kernel: 3, Major: 8, Minor: 20}, + VersionInfo{Kernel: 3, Major: 8, Minor: 0}, + 1) + assertKernelVersion(t, + VersionInfo{Kernel: 3, Major: 8, Minor: 0}, + VersionInfo{Kernel: 3, Major: 8, Minor: 20}, + -1) +} diff --git a/Godeps/_workspace/src/github.com/docker/docker/pkg/parsers/kernel/kernel_windows.go b/Godeps/_workspace/src/github.com/docker/docker/pkg/parsers/kernel/kernel_windows.go new file mode 100644 index 000000000..85ca250c9 --- /dev/null +++ b/Godeps/_workspace/src/github.com/docker/docker/pkg/parsers/kernel/kernel_windows.go @@ -0,0 +1,67 @@ +package kernel + +import ( + "fmt" + "syscall" + "unsafe" +) + +// VersionInfo holds information about the kernel. +type VersionInfo struct { + kvi string // Version of the kernel (e.g. 6.1.7601.17592 -> 6) + major int // Major part of the kernel version (e.g. 6.1.7601.17592 -> 1) + minor int // Minor part of the kernel version (e.g. 6.1.7601.17592 -> 7601) + build int // Build number of the kernel version (e.g. 6.1.7601.17592 -> 17592) +} + +func (k *VersionInfo) String() string { + return fmt.Sprintf("%d.%d %d (%s)", k.major, k.minor, k.build, k.kvi) +} + +// GetKernelVersion gets the current kernel version. +func GetKernelVersion() (*VersionInfo, error) { + + var ( + h syscall.Handle + dwVersion uint32 + err error + ) + + KVI := &VersionInfo{"Unknown", 0, 0, 0} + + if err = syscall.RegOpenKeyEx(syscall.HKEY_LOCAL_MACHINE, + syscall.StringToUTF16Ptr(`SOFTWARE\\Microsoft\\Windows NT\\CurrentVersion\\`), + 0, + syscall.KEY_READ, + &h); err != nil { + return KVI, err + } + defer syscall.RegCloseKey(h) + + var buf [1 << 10]uint16 + var typ uint32 + n := uint32(len(buf) * 2) // api expects array of bytes, not uint16 + + if err = syscall.RegQueryValueEx(h, + syscall.StringToUTF16Ptr("BuildLabEx"), + nil, + &typ, + (*byte)(unsafe.Pointer(&buf[0])), + &n); err != nil { + return KVI, err + } + + KVI.kvi = syscall.UTF16ToString(buf[:]) + + // Important - docker.exe MUST be manifested for this API to return + // the correct information. + if dwVersion, err = syscall.GetVersion(); err != nil { + return KVI, err + } + + KVI.major = int(dwVersion & 0xFF) + KVI.minor = int((dwVersion & 0XFF00) >> 8) + KVI.build = int((dwVersion & 0xFFFF0000) >> 16) + + return KVI, nil +} diff --git a/Godeps/_workspace/src/github.com/docker/docker/pkg/parsers/kernel/uname_linux.go b/Godeps/_workspace/src/github.com/docker/docker/pkg/parsers/kernel/uname_linux.go new file mode 100644 index 000000000..7d12fcbd9 --- /dev/null +++ b/Godeps/_workspace/src/github.com/docker/docker/pkg/parsers/kernel/uname_linux.go @@ -0,0 +1,19 @@ +package kernel + +import ( + "syscall" +) + +// Utsname represents the system name structure. +// It is passthgrouh for syscall.Utsname in order to make it portable with +// other platforms where it is not available. +type Utsname syscall.Utsname + +func uname() (*syscall.Utsname, error) { + uts := &syscall.Utsname{} + + if err := syscall.Uname(uts); err != nil { + return nil, err + } + return uts, nil +} diff --git a/Godeps/_workspace/src/github.com/docker/docker/pkg/parsers/kernel/uname_unsupported.go b/Godeps/_workspace/src/github.com/docker/docker/pkg/parsers/kernel/uname_unsupported.go new file mode 100644 index 000000000..79c66b322 --- /dev/null +++ b/Godeps/_workspace/src/github.com/docker/docker/pkg/parsers/kernel/uname_unsupported.go @@ -0,0 +1,18 @@ +// +build !linux + +package kernel + +import ( + "errors" +) + +// Utsname represents the system name structure. +// It is defined here to make it portable as it is available on linux but not +// on windows. +type Utsname struct { + Release [65]byte +} + +func uname() (*Utsname, error) { + return nil, errors.New("Kernel version detection is available only on linux") +} diff --git a/Godeps/_workspace/src/github.com/docker/docker/pkg/proxy/network_proxy_test.go b/Godeps/_workspace/src/github.com/docker/docker/pkg/proxy/network_proxy_test.go new file mode 100644 index 000000000..9e382567c --- /dev/null +++ b/Godeps/_workspace/src/github.com/docker/docker/pkg/proxy/network_proxy_test.go @@ -0,0 +1,216 @@ +package proxy + +import ( + "bytes" + "fmt" + "io" + "net" + "strings" + "testing" + "time" +) + +var testBuf = []byte("Buffalo buffalo Buffalo buffalo buffalo buffalo Buffalo buffalo") +var testBufSize = len(testBuf) + +type EchoServer interface { + Run() + Close() + LocalAddr() net.Addr +} + +type TCPEchoServer struct { + listener net.Listener + testCtx *testing.T +} + +type UDPEchoServer struct { + conn net.PacketConn + testCtx *testing.T +} + +func NewEchoServer(t *testing.T, proto, address string) EchoServer { + var server EchoServer + if strings.HasPrefix(proto, "tcp") { + listener, err := net.Listen(proto, address) + if err != nil { + t.Fatal(err) + } + server = &TCPEchoServer{listener: listener, testCtx: t} + } else { + socket, err := net.ListenPacket(proto, address) + if err != nil { + t.Fatal(err) + } + server = &UDPEchoServer{conn: socket, testCtx: t} + } + return server +} + +func (server *TCPEchoServer) Run() { + go func() { + for { + client, err := server.listener.Accept() + if err != nil { + return + } + go func(client net.Conn) { + if _, err := io.Copy(client, client); err != nil { + server.testCtx.Logf("can't echo to the client: %v\n", err.Error()) + } + client.Close() + }(client) + } + }() +} + +func (server *TCPEchoServer) LocalAddr() net.Addr { return server.listener.Addr() } +func (server *TCPEchoServer) Close() { server.listener.Addr() } + +func (server *UDPEchoServer) Run() { + go func() { + readBuf := make([]byte, 1024) + for { + read, from, err := server.conn.ReadFrom(readBuf) + if err != nil { + return + } + for i := 0; i != read; { + written, err := server.conn.WriteTo(readBuf[i:read], from) + if err != nil { + break + } + i += written + } + } + }() +} + +func (server *UDPEchoServer) LocalAddr() net.Addr { return server.conn.LocalAddr() } +func (server *UDPEchoServer) Close() { server.conn.Close() } + +func testProxyAt(t *testing.T, proto string, proxy Proxy, addr string) { + defer proxy.Close() + go proxy.Run() + client, err := net.Dial(proto, addr) + if err != nil { + t.Fatalf("Can't connect to the proxy: %v", err) + } + defer client.Close() + client.SetDeadline(time.Now().Add(10 * time.Second)) + if _, err = client.Write(testBuf); err != nil { + t.Fatal(err) + } + recvBuf := make([]byte, testBufSize) + if _, err = client.Read(recvBuf); err != nil { + t.Fatal(err) + } + if !bytes.Equal(testBuf, recvBuf) { + t.Fatal(fmt.Errorf("Expected [%v] but got [%v]", testBuf, recvBuf)) + } +} + +func testProxy(t *testing.T, proto string, proxy Proxy) { + testProxyAt(t, proto, proxy, proxy.FrontendAddr().String()) +} + +func TestTCP4Proxy(t *testing.T) { + backend := NewEchoServer(t, "tcp", "127.0.0.1:0") + defer backend.Close() + backend.Run() + frontendAddr := &net.TCPAddr{IP: net.IPv4(127, 0, 0, 1), Port: 0} + proxy, err := NewProxy(frontendAddr, backend.LocalAddr()) + if err != nil { + t.Fatal(err) + } + testProxy(t, "tcp", proxy) +} + +func TestTCP6Proxy(t *testing.T) { + backend := NewEchoServer(t, "tcp", "[::1]:0") + defer backend.Close() + backend.Run() + frontendAddr := &net.TCPAddr{IP: net.IPv6loopback, Port: 0} + proxy, err := NewProxy(frontendAddr, backend.LocalAddr()) + if err != nil { + t.Fatal(err) + } + testProxy(t, "tcp", proxy) +} + +func TestTCPDualStackProxy(t *testing.T) { + // If I understand `godoc -src net favoriteAddrFamily` (used by the + // net.Listen* functions) correctly this should work, but it doesn't. + t.Skip("No support for dual stack yet") + backend := NewEchoServer(t, "tcp", "[::1]:0") + defer backend.Close() + backend.Run() + frontendAddr := &net.TCPAddr{IP: net.IPv6loopback, Port: 0} + proxy, err := NewProxy(frontendAddr, backend.LocalAddr()) + if err != nil { + t.Fatal(err) + } + ipv4ProxyAddr := &net.TCPAddr{ + IP: net.IPv4(127, 0, 0, 1), + Port: proxy.FrontendAddr().(*net.TCPAddr).Port, + } + testProxyAt(t, "tcp", proxy, ipv4ProxyAddr.String()) +} + +func TestUDP4Proxy(t *testing.T) { + backend := NewEchoServer(t, "udp", "127.0.0.1:0") + defer backend.Close() + backend.Run() + frontendAddr := &net.UDPAddr{IP: net.IPv4(127, 0, 0, 1), Port: 0} + proxy, err := NewProxy(frontendAddr, backend.LocalAddr()) + if err != nil { + t.Fatal(err) + } + testProxy(t, "udp", proxy) +} + +func TestUDP6Proxy(t *testing.T) { + backend := NewEchoServer(t, "udp", "[::1]:0") + defer backend.Close() + backend.Run() + frontendAddr := &net.UDPAddr{IP: net.IPv6loopback, Port: 0} + proxy, err := NewProxy(frontendAddr, backend.LocalAddr()) + if err != nil { + t.Fatal(err) + } + testProxy(t, "udp", proxy) +} + +func TestUDPWriteError(t *testing.T) { + frontendAddr := &net.UDPAddr{IP: net.IPv4(127, 0, 0, 1), Port: 0} + // Hopefully, this port will be free: */ + backendAddr := &net.UDPAddr{IP: net.IPv4(127, 0, 0, 1), Port: 25587} + proxy, err := NewProxy(frontendAddr, backendAddr) + if err != nil { + t.Fatal(err) + } + defer proxy.Close() + go proxy.Run() + client, err := net.Dial("udp", "127.0.0.1:25587") + if err != nil { + t.Fatalf("Can't connect to the proxy: %v", err) + } + defer client.Close() + // Make sure the proxy doesn't stop when there is no actual backend: + client.Write(testBuf) + client.Write(testBuf) + backend := NewEchoServer(t, "udp", "127.0.0.1:25587") + defer backend.Close() + backend.Run() + client.SetDeadline(time.Now().Add(10 * time.Second)) + if _, err = client.Write(testBuf); err != nil { + t.Fatal(err) + } + recvBuf := make([]byte, testBufSize) + if _, err = client.Read(recvBuf); err != nil { + t.Fatal(err) + } + if !bytes.Equal(testBuf, recvBuf) { + t.Fatal(fmt.Errorf("Expected [%v] but got [%v]", testBuf, recvBuf)) + } +} diff --git a/Godeps/_workspace/src/github.com/docker/docker/pkg/proxy/proxy.go b/Godeps/_workspace/src/github.com/docker/docker/pkg/proxy/proxy.go new file mode 100644 index 000000000..4e24e5f6a --- /dev/null +++ b/Godeps/_workspace/src/github.com/docker/docker/pkg/proxy/proxy.go @@ -0,0 +1,37 @@ +// Package proxy provides a network Proxy interface and implementations for TCP +// and UDP. +package proxy + +import ( + "fmt" + "net" +) + +// Proxy defines the behavior of a proxy. It forwards traffic back and forth +// between two endpoints : the frontend and the backend. +// It can be used to do software port-mapping between two addresses. +// e.g. forward all traffic between the frontend (host) 127.0.0.1:3000 +// to the backend (container) at 172.17.42.108:4000. +type Proxy interface { + // Run starts forwarding traffic back and forth between the front + // and back-end addresses. + Run() + // Close stops forwarding traffic and close both ends of the Proxy. + Close() + // FrontendAddr returns the address on which the proxy is listening. + FrontendAddr() net.Addr + // BackendAddr returns the proxied address. + BackendAddr() net.Addr +} + +// NewProxy creates a Proxy according to the specified frontendAddr and backendAddr. +func NewProxy(frontendAddr, backendAddr net.Addr) (Proxy, error) { + switch frontendAddr.(type) { + case *net.UDPAddr: + return NewUDPProxy(frontendAddr.(*net.UDPAddr), backendAddr.(*net.UDPAddr)) + case *net.TCPAddr: + return NewTCPProxy(frontendAddr.(*net.TCPAddr), backendAddr.(*net.TCPAddr)) + default: + panic(fmt.Errorf("Unsupported protocol")) + } +} diff --git a/Godeps/_workspace/src/github.com/docker/docker/pkg/proxy/stub_proxy.go b/Godeps/_workspace/src/github.com/docker/docker/pkg/proxy/stub_proxy.go new file mode 100644 index 000000000..571749e46 --- /dev/null +++ b/Godeps/_workspace/src/github.com/docker/docker/pkg/proxy/stub_proxy.go @@ -0,0 +1,31 @@ +package proxy + +import ( + "net" +) + +// StubProxy is a proxy that is a stub (does nothing). +type StubProxy struct { + frontendAddr net.Addr + backendAddr net.Addr +} + +// Run does nothing. +func (p *StubProxy) Run() {} + +// Close does nothing. +func (p *StubProxy) Close() {} + +// FrontendAddr returns the frontend address. +func (p *StubProxy) FrontendAddr() net.Addr { return p.frontendAddr } + +// BackendAddr returns the backend address. +func (p *StubProxy) BackendAddr() net.Addr { return p.backendAddr } + +// NewStubProxy creates a new StubProxy +func NewStubProxy(frontendAddr, backendAddr net.Addr) (Proxy, error) { + return &StubProxy{ + frontendAddr: frontendAddr, + backendAddr: backendAddr, + }, nil +} diff --git a/Godeps/_workspace/src/github.com/docker/docker/pkg/proxy/tcp_proxy.go b/Godeps/_workspace/src/github.com/docker/docker/pkg/proxy/tcp_proxy.go new file mode 100644 index 000000000..3cd742af7 --- /dev/null +++ b/Godeps/_workspace/src/github.com/docker/docker/pkg/proxy/tcp_proxy.go @@ -0,0 +1,99 @@ +package proxy + +import ( + "io" + "net" + "syscall" + + "github.com/Sirupsen/logrus" +) + +// TCPProxy is a proxy for TCP connections. It implements the Proxy interface to +// handle TCP traffic forwarding between the frontend and backend addresses. +type TCPProxy struct { + listener *net.TCPListener + frontendAddr *net.TCPAddr + backendAddr *net.TCPAddr +} + +// NewTCPProxy creates a new TCPProxy. +func NewTCPProxy(frontendAddr, backendAddr *net.TCPAddr) (*TCPProxy, error) { + listener, err := net.ListenTCP("tcp", frontendAddr) + if err != nil { + return nil, err + } + // If the port in frontendAddr was 0 then ListenTCP will have a picked + // a port to listen on, hence the call to Addr to get that actual port: + return &TCPProxy{ + listener: listener, + frontendAddr: listener.Addr().(*net.TCPAddr), + backendAddr: backendAddr, + }, nil +} + +func (proxy *TCPProxy) clientLoop(client *net.TCPConn, quit chan bool) { + backend, err := net.DialTCP("tcp", nil, proxy.backendAddr) + if err != nil { + logrus.Printf("Can't forward traffic to backend tcp/%v: %s\n", proxy.backendAddr, err) + client.Close() + return + } + + event := make(chan int64) + var broker = func(to, from *net.TCPConn) { + written, err := io.Copy(to, from) + if err != nil { + // If the socket we are writing to is shutdown with + // SHUT_WR, forward it to the other end of the pipe: + if err, ok := err.(*net.OpError); ok && err.Err == syscall.EPIPE { + from.CloseWrite() + } + } + to.CloseRead() + event <- written + } + + go broker(client, backend) + go broker(backend, client) + + var transferred int64 + for i := 0; i < 2; i++ { + select { + case written := <-event: + transferred += written + case <-quit: + // Interrupt the two brokers and "join" them. + client.Close() + backend.Close() + for ; i < 2; i++ { + transferred += <-event + } + return + } + } + client.Close() + backend.Close() +} + +// Run starts forwarding the traffic using TCP. +func (proxy *TCPProxy) Run() { + quit := make(chan bool) + defer close(quit) + for { + client, err := proxy.listener.Accept() + if err != nil { + logrus.Printf("Stopping proxy on tcp/%v for tcp/%v (%s)", proxy.frontendAddr, proxy.backendAddr, err) + return + } + go proxy.clientLoop(client.(*net.TCPConn), quit) + } +} + +// Close stops forwarding the traffic. +func (proxy *TCPProxy) Close() { proxy.listener.Close() } + +// FrontendAddr returns the TCP address on which the proxy is listening. +func (proxy *TCPProxy) FrontendAddr() net.Addr { return proxy.frontendAddr } + +// BackendAddr returns the TCP proxied address. +func (proxy *TCPProxy) BackendAddr() net.Addr { return proxy.backendAddr } diff --git a/Godeps/_workspace/src/github.com/docker/docker/pkg/proxy/udp_proxy.go b/Godeps/_workspace/src/github.com/docker/docker/pkg/proxy/udp_proxy.go new file mode 100644 index 000000000..b8375c374 --- /dev/null +++ b/Godeps/_workspace/src/github.com/docker/docker/pkg/proxy/udp_proxy.go @@ -0,0 +1,169 @@ +package proxy + +import ( + "encoding/binary" + "net" + "strings" + "sync" + "syscall" + "time" + + "github.com/Sirupsen/logrus" +) + +const ( + // UDPConnTrackTimeout is the timeout used for UDP connection tracking + UDPConnTrackTimeout = 90 * time.Second + // UDPBufSize is the buffer size for the UDP proxy + UDPBufSize = 65507 +) + +// A net.Addr where the IP is split into two fields so you can use it as a key +// in a map: +type connTrackKey struct { + IPHigh uint64 + IPLow uint64 + Port int +} + +func newConnTrackKey(addr *net.UDPAddr) *connTrackKey { + if len(addr.IP) == net.IPv4len { + return &connTrackKey{ + IPHigh: 0, + IPLow: uint64(binary.BigEndian.Uint32(addr.IP)), + Port: addr.Port, + } + } + return &connTrackKey{ + IPHigh: binary.BigEndian.Uint64(addr.IP[:8]), + IPLow: binary.BigEndian.Uint64(addr.IP[8:]), + Port: addr.Port, + } +} + +type connTrackMap map[connTrackKey]*net.UDPConn + +// UDPProxy is proxy for which handles UDP datagrams. It implements the Proxy +// interface to handle UDP traffic forwarding between the frontend and backend +// addresses. +type UDPProxy struct { + listener *net.UDPConn + frontendAddr *net.UDPAddr + backendAddr *net.UDPAddr + connTrackTable connTrackMap + connTrackLock sync.Mutex +} + +// NewUDPProxy creates a new UDPProxy. +func NewUDPProxy(frontendAddr, backendAddr *net.UDPAddr) (*UDPProxy, error) { + listener, err := net.ListenUDP("udp", frontendAddr) + if err != nil { + return nil, err + } + return &UDPProxy{ + listener: listener, + frontendAddr: listener.LocalAddr().(*net.UDPAddr), + backendAddr: backendAddr, + connTrackTable: make(connTrackMap), + }, nil +} + +func (proxy *UDPProxy) replyLoop(proxyConn *net.UDPConn, clientAddr *net.UDPAddr, clientKey *connTrackKey) { + defer func() { + proxy.connTrackLock.Lock() + delete(proxy.connTrackTable, *clientKey) + proxy.connTrackLock.Unlock() + proxyConn.Close() + }() + + readBuf := make([]byte, UDPBufSize) + for { + proxyConn.SetReadDeadline(time.Now().Add(UDPConnTrackTimeout)) + again: + read, err := proxyConn.Read(readBuf) + if err != nil { + if err, ok := err.(*net.OpError); ok && err.Err == syscall.ECONNREFUSED { + // This will happen if the last write failed + // (e.g: nothing is actually listening on the + // proxied port on the container), ignore it + // and continue until UDPConnTrackTimeout + // expires: + goto again + } + return + } + for i := 0; i != read; { + written, err := proxy.listener.WriteToUDP(readBuf[i:read], clientAddr) + if err != nil { + return + } + i += written + } + } +} + +// Run starts forwarding the traffic using UDP. +func (proxy *UDPProxy) Run() { + readBuf := make([]byte, UDPBufSize) + for { + read, from, err := proxy.listener.ReadFromUDP(readBuf) + if err != nil { + // NOTE: Apparently ReadFrom doesn't return + // ECONNREFUSED like Read do (see comment in + // UDPProxy.replyLoop) + if !isClosedError(err) { + logrus.Printf("Stopping proxy on udp/%v for udp/%v (%s)", proxy.frontendAddr, proxy.backendAddr, err) + } + break + } + + fromKey := newConnTrackKey(from) + proxy.connTrackLock.Lock() + proxyConn, hit := proxy.connTrackTable[*fromKey] + if !hit { + proxyConn, err = net.DialUDP("udp", nil, proxy.backendAddr) + if err != nil { + logrus.Printf("Can't proxy a datagram to udp/%s: %s\n", proxy.backendAddr, err) + proxy.connTrackLock.Unlock() + continue + } + proxy.connTrackTable[*fromKey] = proxyConn + go proxy.replyLoop(proxyConn, from, fromKey) + } + proxy.connTrackLock.Unlock() + for i := 0; i != read; { + written, err := proxyConn.Write(readBuf[i:read]) + if err != nil { + logrus.Printf("Can't proxy a datagram to udp/%s: %s\n", proxy.backendAddr, err) + break + } + i += written + } + } +} + +// Close stops forwarding the traffic. +func (proxy *UDPProxy) Close() { + proxy.listener.Close() + proxy.connTrackLock.Lock() + defer proxy.connTrackLock.Unlock() + for _, conn := range proxy.connTrackTable { + conn.Close() + } +} + +// FrontendAddr returns the UDP address on which the proxy is listening. +func (proxy *UDPProxy) FrontendAddr() net.Addr { return proxy.frontendAddr } + +// BackendAddr returns the proxied UDP address. +func (proxy *UDPProxy) BackendAddr() net.Addr { return proxy.backendAddr } + +func isClosedError(err error) bool { + /* This comparison is ugly, but unfortunately, net.go doesn't export errClosing. + * See: + * http://golang.org/src/pkg/net/net.go + * https://code.google.com/p/go/issues/detail?id=4337 + * https://groups.google.com/forum/#!msg/golang-nuts/0_aaCvBmOcM/SptmDyX1XJMJ + */ + return strings.HasSuffix(err.Error(), "use of closed network connection") +} diff --git a/Godeps/_workspace/src/github.com/docker/docker/pkg/random/random.go b/Godeps/_workspace/src/github.com/docker/docker/pkg/random/random.go new file mode 100644 index 000000000..e560aff11 --- /dev/null +++ b/Godeps/_workspace/src/github.com/docker/docker/pkg/random/random.go @@ -0,0 +1,71 @@ +package random + +import ( + cryptorand "crypto/rand" + "io" + "math" + "math/big" + "math/rand" + "sync" + "time" +) + +// Rand is a global *rand.Rand instance, which initilized with NewSource() source. +var Rand = rand.New(NewSource()) + +// Reader is a global, shared instance of a pseudorandom bytes generator. +// It doesn't consume entropy. +var Reader io.Reader = &reader{rnd: Rand} + +// copypaste from standard math/rand +type lockedSource struct { + lk sync.Mutex + src rand.Source +} + +func (r *lockedSource) Int63() (n int64) { + r.lk.Lock() + n = r.src.Int63() + r.lk.Unlock() + return +} + +func (r *lockedSource) Seed(seed int64) { + r.lk.Lock() + r.src.Seed(seed) + r.lk.Unlock() +} + +// NewSource returns math/rand.Source safe for concurrent use and initialized +// with current unix-nano timestamp +func NewSource() rand.Source { + var seed int64 + if cryptoseed, err := cryptorand.Int(cryptorand.Reader, big.NewInt(math.MaxInt64)); err != nil { + // This should not happen, but worst-case fallback to time-based seed. + seed = time.Now().UnixNano() + } else { + seed = cryptoseed.Int64() + } + return &lockedSource{ + src: rand.NewSource(seed), + } +} + +type reader struct { + rnd *rand.Rand +} + +func (r *reader) Read(b []byte) (int, error) { + i := 0 + for { + val := r.rnd.Int63() + for val > 0 { + b[i] = byte(val) + i++ + if i == len(b) { + return i, nil + } + val >>= 8 + } + } +} diff --git a/Godeps/_workspace/src/github.com/docker/docker/pkg/random/random_test.go b/Godeps/_workspace/src/github.com/docker/docker/pkg/random/random_test.go new file mode 100644 index 000000000..cf405f78c --- /dev/null +++ b/Godeps/_workspace/src/github.com/docker/docker/pkg/random/random_test.go @@ -0,0 +1,22 @@ +package random + +import ( + "math/rand" + "sync" + "testing" +) + +// for go test -v -race +func TestConcurrency(t *testing.T) { + rnd := rand.New(NewSource()) + var wg sync.WaitGroup + + for i := 0; i < 10; i++ { + wg.Add(1) + go func() { + rnd.Int63() + wg.Done() + }() + } + wg.Wait() +} diff --git a/Godeps/_workspace/src/github.com/docker/docker/pkg/reexec/README.md b/Godeps/_workspace/src/github.com/docker/docker/pkg/reexec/README.md new file mode 100644 index 000000000..45592ce85 --- /dev/null +++ b/Godeps/_workspace/src/github.com/docker/docker/pkg/reexec/README.md @@ -0,0 +1,5 @@ +## reexec + +The `reexec` package facilitates the busybox style reexec of the docker binary that we require because +of the forking limitations of using Go. Handlers can be registered with a name and the argv 0 of +the exec of the binary will be used to find and execute custom init paths. diff --git a/Godeps/_workspace/src/github.com/docker/docker/pkg/reexec/command_freebsd.go b/Godeps/_workspace/src/github.com/docker/docker/pkg/reexec/command_freebsd.go new file mode 100644 index 000000000..c7f797a5f --- /dev/null +++ b/Godeps/_workspace/src/github.com/docker/docker/pkg/reexec/command_freebsd.go @@ -0,0 +1,23 @@ +// +build freebsd + +package reexec + +import ( + "os/exec" +) + +// Self returns the path to the current process's binary. +// Uses os.Args[0]. +func Self() string { + return naiveSelf() +} + +// Command returns *exec.Cmd which have Path as current binary. +// For example if current binary is "docker" at "/usr/bin/", then cmd.Path will +// be set to "/usr/bin/docker". +func Command(args ...string) *exec.Cmd { + return &exec.Cmd{ + Path: Self(), + Args: args, + } +} diff --git a/Godeps/_workspace/src/github.com/docker/docker/pkg/reexec/command_linux.go b/Godeps/_workspace/src/github.com/docker/docker/pkg/reexec/command_linux.go new file mode 100644 index 000000000..3c3a73a9d --- /dev/null +++ b/Godeps/_workspace/src/github.com/docker/docker/pkg/reexec/command_linux.go @@ -0,0 +1,28 @@ +// +build linux + +package reexec + +import ( + "os/exec" + "syscall" +) + +// Self returns the path to the current process's binary. +// Returns "/proc/self/exe". +func Self() string { + return "/proc/self/exe" +} + +// Command returns *exec.Cmd which have Path as current binary. Also it setting +// SysProcAttr.Pdeathsig to SIGTERM. +// This will use the in-memory version (/proc/self/exe) of the current binary, +// it is thus safe to delete or replace the on-disk binary (os.Args[0]). +func Command(args ...string) *exec.Cmd { + return &exec.Cmd{ + Path: Self(), + Args: args, + SysProcAttr: &syscall.SysProcAttr{ + Pdeathsig: syscall.SIGTERM, + }, + } +} diff --git a/Godeps/_workspace/src/github.com/docker/docker/pkg/reexec/command_unsupported.go b/Godeps/_workspace/src/github.com/docker/docker/pkg/reexec/command_unsupported.go new file mode 100644 index 000000000..ad4ea38eb --- /dev/null +++ b/Godeps/_workspace/src/github.com/docker/docker/pkg/reexec/command_unsupported.go @@ -0,0 +1,12 @@ +// +build !linux,!windows,!freebsd + +package reexec + +import ( + "os/exec" +) + +// Command is unsupported on operating systems apart from Linux and Windows. +func Command(args ...string) *exec.Cmd { + return nil +} diff --git a/Godeps/_workspace/src/github.com/docker/docker/pkg/reexec/command_windows.go b/Godeps/_workspace/src/github.com/docker/docker/pkg/reexec/command_windows.go new file mode 100644 index 000000000..8d65e0ae1 --- /dev/null +++ b/Godeps/_workspace/src/github.com/docker/docker/pkg/reexec/command_windows.go @@ -0,0 +1,23 @@ +// +build windows + +package reexec + +import ( + "os/exec" +) + +// Self returns the path to the current process's binary. +// Uses os.Args[0]. +func Self() string { + return naiveSelf() +} + +// Command returns *exec.Cmd which have Path as current binary. +// For example if current binary is "docker.exe" at "C:\", then cmd.Path will +// be set to "C:\docker.exe". +func Command(args ...string) *exec.Cmd { + return &exec.Cmd{ + Path: Self(), + Args: args, + } +} diff --git a/Godeps/_workspace/src/github.com/docker/docker/pkg/reexec/reexec.go b/Godeps/_workspace/src/github.com/docker/docker/pkg/reexec/reexec.go new file mode 100644 index 000000000..20491e05d --- /dev/null +++ b/Godeps/_workspace/src/github.com/docker/docker/pkg/reexec/reexec.go @@ -0,0 +1,47 @@ +package reexec + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" +) + +var registeredInitializers = make(map[string]func()) + +// Register adds an initialization func under the specified name +func Register(name string, initializer func()) { + if _, exists := registeredInitializers[name]; exists { + panic(fmt.Sprintf("reexec func already registred under name %q", name)) + } + + registeredInitializers[name] = initializer +} + +// Init is called as the first part of the exec process and returns true if an +// initialization function was called. +func Init() bool { + initializer, exists := registeredInitializers[os.Args[0]] + if exists { + initializer() + + return true + } + return false +} + +func naiveSelf() string { + name := os.Args[0] + if filepath.Base(name) == name { + if lp, err := exec.LookPath(name); err == nil { + return lp + } + } + // handle conversion of relative paths to absolute + if absName, err := filepath.Abs(name); err == nil { + return absName + } + // if we coudn't get absolute name, return original + // (NOTE: Go only errors on Abs() if os.Getwd fails) + return name +} diff --git a/Godeps/_workspace/src/github.com/docker/docker/pkg/stringid/README.md b/Godeps/_workspace/src/github.com/docker/docker/pkg/stringid/README.md new file mode 100644 index 000000000..37a5098fd --- /dev/null +++ b/Godeps/_workspace/src/github.com/docker/docker/pkg/stringid/README.md @@ -0,0 +1 @@ +This package provides helper functions for dealing with string identifiers diff --git a/Godeps/_workspace/src/github.com/docker/docker/pkg/stringid/stringid.go b/Godeps/_workspace/src/github.com/docker/docker/pkg/stringid/stringid.go new file mode 100644 index 000000000..ab1f9d474 --- /dev/null +++ b/Godeps/_workspace/src/github.com/docker/docker/pkg/stringid/stringid.go @@ -0,0 +1,67 @@ +// Package stringid provides helper functions for dealing with string identifiers +package stringid + +import ( + "crypto/rand" + "encoding/hex" + "io" + "regexp" + "strconv" + + "github.com/docker/docker/pkg/random" +) + +const shortLen = 12 + +var validShortID = regexp.MustCompile("^[a-z0-9]{12}$") + +// IsShortID determines if an arbitrary string *looks like* a short ID. +func IsShortID(id string) bool { + return validShortID.MatchString(id) +} + +// TruncateID returns a shorthand version of a string identifier for convenience. +// A collision with other shorthands is very unlikely, but possible. +// In case of a collision a lookup with TruncIndex.Get() will fail, and the caller +// will need to use a langer prefix, or the full-length Id. +func TruncateID(id string) string { + trimTo := shortLen + if len(id) < shortLen { + trimTo = len(id) + } + return id[:trimTo] +} + +func generateID(crypto bool) string { + b := make([]byte, 32) + var r io.Reader = random.Reader + if crypto { + r = rand.Reader + } + for { + if _, err := io.ReadFull(r, b); err != nil { + panic(err) // This shouldn't happen + } + id := hex.EncodeToString(b) + // if we try to parse the truncated for as an int and we don't have + // an error then the value is all numberic and causes issues when + // used as a hostname. ref #3869 + if _, err := strconv.ParseInt(TruncateID(id), 10, 64); err == nil { + continue + } + return id + } +} + +// GenerateRandomID returns an unique id. +func GenerateRandomID() string { + return generateID(true) + +} + +// GenerateNonCryptoID generates unique id without using cryptographically +// secure sources of random. +// It helps you to save entropy. +func GenerateNonCryptoID() string { + return generateID(false) +} diff --git a/Godeps/_workspace/src/github.com/docker/docker/pkg/stringid/stringid_test.go b/Godeps/_workspace/src/github.com/docker/docker/pkg/stringid/stringid_test.go new file mode 100644 index 000000000..bcb136549 --- /dev/null +++ b/Godeps/_workspace/src/github.com/docker/docker/pkg/stringid/stringid_test.go @@ -0,0 +1,56 @@ +package stringid + +import ( + "strings" + "testing" +) + +func TestGenerateRandomID(t *testing.T) { + id := GenerateRandomID() + + if len(id) != 64 { + t.Fatalf("Id returned is incorrect: %s", id) + } +} + +func TestShortenId(t *testing.T) { + id := GenerateRandomID() + truncID := TruncateID(id) + if len(truncID) != 12 { + t.Fatalf("Id returned is incorrect: truncate on %s returned %s", id, truncID) + } +} + +func TestShortenIdEmpty(t *testing.T) { + id := "" + truncID := TruncateID(id) + if len(truncID) > len(id) { + t.Fatalf("Id returned is incorrect: truncate on %s returned %s", id, truncID) + } +} + +func TestShortenIdInvalid(t *testing.T) { + id := "1234" + truncID := TruncateID(id) + if len(truncID) != len(id) { + t.Fatalf("Id returned is incorrect: truncate on %s returned %s", id, truncID) + } +} + +func TestIsShortIDNonHex(t *testing.T) { + id := "some non-hex value" + if IsShortID(id) { + t.Fatalf("%s is not a short ID", id) + } +} + +func TestIsShortIDNotCorrectSize(t *testing.T) { + id := strings.Repeat("a", shortLen+1) + if IsShortID(id) { + t.Fatalf("%s is not a short ID", id) + } + id = strings.Repeat("a", shortLen-1) + if IsShortID(id) { + t.Fatalf("%s is not a short ID", id) + } +} diff --git a/Godeps/_workspace/src/github.com/docker/libkv/.travis.yml b/Godeps/_workspace/src/github.com/docker/libkv/.travis.yml new file mode 100644 index 000000000..ae2704930 --- /dev/null +++ b/Godeps/_workspace/src/github.com/docker/libkv/.travis.yml @@ -0,0 +1,34 @@ +language: go + +go: + - 1.3 +# - 1.4 +# see https://github.com/moovweb/gvm/pull/116 for why Go 1.4 is currently disabled + +# let us have speedy Docker-based Travis workers +sudo: false + +before_install: + # Symlink below is needed for Travis CI to work correctly on personal forks of libkv + - ln -s $HOME/gopath/src/github.com/${TRAVIS_REPO_SLUG///libkv/} $HOME/gopath/src/github.com/docker + - go get golang.org/x/tools/cmd/vet + - go get golang.org/x/tools/cmd/cover + - go get github.com/mattn/goveralls + - go get github.com/golang/lint/golint + - go get github.com/GeertJohan/fgt + +before_script: + - script/travis_consul.sh 0.5.2 + - script/travis_etcd.sh 2.0.11 + - script/travis_zk.sh 3.4.6 + +script: + - ./consul agent -server -bootstrap-expect 1 -data-dir /tmp/consul -config-file=./config.json 1>/dev/null & + - ./etcd/etcd --listen-client-urls 'http://0.0.0.0:4001' --advertise-client-urls 'http://127.0.0.1:4001' >/dev/null 2>&1 & + - ./zk/bin/zkServer.sh start ./zk/conf/zoo.cfg 1> /dev/null + - script/validate-gofmt + - go vet ./... + - fgt golint ./... + - go test -v -race ./... + - script/coverage + - goveralls -service=travis-ci -coverprofile=goverage.report diff --git a/Godeps/_workspace/src/github.com/docker/libkv/LICENSE b/Godeps/_workspace/src/github.com/docker/libkv/LICENSE new file mode 100644 index 000000000..9e4bd4dbe --- /dev/null +++ b/Godeps/_workspace/src/github.com/docker/libkv/LICENSE @@ -0,0 +1,191 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + Copyright 2014-2015 Docker, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/Godeps/_workspace/src/github.com/docker/libkv/README.md b/Godeps/_workspace/src/github.com/docker/libkv/README.md new file mode 100644 index 000000000..d09b34c1e --- /dev/null +++ b/Godeps/_workspace/src/github.com/docker/libkv/README.md @@ -0,0 +1,108 @@ +# libkv + +[![GoDoc](https://godoc.org/github.com/docker/libkv?status.png)](https://godoc.org/github.com/docker/libkv) +[![Build Status](https://travis-ci.org/docker/libkv.svg?branch=master)](https://travis-ci.org/docker/libkv) +[![Coverage Status](https://coveralls.io/repos/docker/libkv/badge.svg)](https://coveralls.io/r/docker/libkv) + +`libkv` provides a `Go` native library to store metadata. + +The goal of `libkv` is to abstract common store operations for multiple Key/Value backends and offer the same experience no matter which one of the backend you want to use. + +For example, you can use it to store your metadata or for service discovery to register machines and endpoints inside your cluster. + +You can also easily implement a generic *Leader Election* on top of it (see the [swarm/leadership](https://github.com/docker/swarm/tree/master/leadership) package). + +As of now, `libkv` offers support for `Consul`, `Etcd` and `Zookeeper`. + +## Example of usage + +### Create a new store and use Put/Get + +```go +package main + +import ( + "fmt" + "time" + + "github.com/docker/libkv" + "github.com/docker/libkv/store" + log "github.com/Sirupsen/logrus" +) + +func main() { + client := "localhost:8500" + + // Initialize a new store with consul + kv, err = libkv.NewStore( + store.CONSUL, // or "consul" + []string{client}, + &store.Config{ + ConnectionTimeout: 10*time.Second, + }, + ) + if err != nil { + log.Fatal("Cannot create store consul") + } + + key := "foo" + err = kv.Put(key, []byte("bar"), nil) + if err != nil { + log.Error("Error trying to put value at key `", key, "`") + } + + pair, err := kv.Get(key) + if err != nil { + log.Error("Error trying accessing value at key `", key, "`") + } + + log.Info("value: ", string(pair.Value)) +} +``` + +You can find other usage examples for `libkv` under the `docker/swarm` or `docker/libnetwork` repositories. + +## Details + +You should expect the same experience for basic operations like `Get`/`Put`, etc. + +However calls like `WatchTree` may return different events (or number of events) depending on the backend (for now, `Etcd` and `Consul` will likely return more events than `Zookeeper` that you should triage properly). + +## Create a new storage backend + +A new **storage backend** should include those calls: + +```go +type Store interface { + Put(key string, value []byte, options *WriteOptions) error + Get(key string) (*KVPair, error) + Delete(key string) error + Exists(key string) (bool, error) + Watch(key string, stopCh <-chan struct{}) (<-chan *KVPair, error) + WatchTree(directory string, stopCh <-chan struct{}) (<-chan []*KVPair, error) + NewLock(key string, options *LockOptions) (Locker, error) + List(directory string) ([]*KVPair, error) + DeleteTree(directory string) error + AtomicPut(key string, value []byte, previous *KVPair, options *WriteOptions) (bool, *KVPair, error) + AtomicDelete(key string, previous *KVPair) (bool, error) + Close() +} +``` + +You can get inspiration from existing backends to create a new one. This interface could be subject to changes to improve the experience of using the library and contributing to a new backend. + +##Roadmap + +- Make the API nicer to use (using `options`) +- Provide more options (`consistency` for example) +- Improve performance (remove extras `Get`/`List` operations) +- Add more exhaustive tests +- New backends? + +##Contributing + +Want to hack on libkv? [Docker's contributions guidelines](https://github.com/docker/docker/blob/master/CONTRIBUTING.md) apply. + +##Copyright and license + +Code and documentation copyright 2015 Docker, inc. Code released under the Apache 2.0 license. Docs released under Creative commons. diff --git a/Godeps/_workspace/src/github.com/docker/libkv/libkv.go b/Godeps/_workspace/src/github.com/docker/libkv/libkv.go new file mode 100644 index 000000000..28df703af --- /dev/null +++ b/Godeps/_workspace/src/github.com/docker/libkv/libkv.go @@ -0,0 +1,29 @@ +package libkv + +import ( + "github.com/docker/libkv/store" + "github.com/docker/libkv/store/consul" + "github.com/docker/libkv/store/etcd" + "github.com/docker/libkv/store/zookeeper" +) + +// Initialize creates a new Store object, initializing the client +type Initialize func(addrs []string, options *store.Config) (store.Store, error) + +var ( + // Backend initializers + initializers = map[store.Backend]Initialize{ + store.CONSUL: consul.New, + store.ETCD: etcd.New, + store.ZK: zookeeper.New, + } +) + +// NewStore creates a an instance of store +func NewStore(backend store.Backend, addrs []string, options *store.Config) (store.Store, error) { + if init, exists := initializers[backend]; exists { + return init(addrs, options) + } + + return nil, store.ErrNotSupported +} diff --git a/Godeps/_workspace/src/github.com/docker/libkv/libkv_test.go b/Godeps/_workspace/src/github.com/docker/libkv/libkv_test.go new file mode 100644 index 000000000..8b2ae0e6c --- /dev/null +++ b/Godeps/_workspace/src/github.com/docker/libkv/libkv_test.go @@ -0,0 +1,80 @@ +package libkv + +import ( + "testing" + "time" + + "github.com/docker/libkv/store" + "github.com/docker/libkv/store/consul" + "github.com/docker/libkv/store/etcd" + "github.com/docker/libkv/store/zookeeper" + "github.com/stretchr/testify/assert" +) + +func TestNewStoreConsul(t *testing.T) { + client := "localhost:8500" + + kv, err := NewStore( + store.CONSUL, + []string{client}, + &store.Config{ + ConnectionTimeout: 10 * time.Second, + }, + ) + assert.NoError(t, err) + assert.NotNil(t, kv) + + if _, ok := kv.(*consul.Consul); !ok { + t.Fatal("Error while initializing store consul") + } +} + +func TestNewStoreEtcd(t *testing.T) { + client := "localhost:4001" + + kv, err := NewStore( + store.ETCD, + []string{client}, + &store.Config{ + ConnectionTimeout: 10 * time.Second, + }, + ) + assert.NoError(t, err) + assert.NotNil(t, kv) + + if _, ok := kv.(*etcd.Etcd); !ok { + t.Fatal("Error while initializing store etcd") + } +} + +func TestNewStoreZookeeper(t *testing.T) { + client := "localhost:2181" + + kv, err := NewStore( + store.ZK, + []string{client}, + &store.Config{ + ConnectionTimeout: 10 * time.Second, + }, + ) + assert.NoError(t, err) + assert.NotNil(t, kv) + + if _, ok := kv.(*zookeeper.Zookeeper); !ok { + t.Fatal("Error while initializing store zookeeper") + } +} + +func TestNewStoreUnsupported(t *testing.T) { + client := "localhost:9999" + + kv, err := NewStore( + "unsupported", + []string{client}, + &store.Config{ + ConnectionTimeout: 10 * time.Second, + }, + ) + assert.Error(t, err) + assert.Nil(t, kv) +} diff --git a/Godeps/_workspace/src/github.com/docker/libkv/script/.validate b/Godeps/_workspace/src/github.com/docker/libkv/script/.validate new file mode 100644 index 000000000..3767f4223 --- /dev/null +++ b/Godeps/_workspace/src/github.com/docker/libkv/script/.validate @@ -0,0 +1,33 @@ +#!/bin/bash + +if [ -z "$VALIDATE_UPSTREAM" ]; then + # this is kind of an expensive check, so let's not do this twice if we + # are running more than one validate bundlescript + + VALIDATE_REPO='https://github.com/docker/libkv.git' + VALIDATE_BRANCH='master' + + if [ "$TRAVIS" = 'true' -a "$TRAVIS_PULL_REQUEST" != 'false' ]; then + VALIDATE_REPO="https://github.com/${TRAVIS_REPO_SLUG}.git" + VALIDATE_BRANCH="${TRAVIS_BRANCH}" + fi + + VALIDATE_HEAD="$(git rev-parse --verify HEAD)" + + git fetch -q "$VALIDATE_REPO" "refs/heads/$VALIDATE_BRANCH" + VALIDATE_UPSTREAM="$(git rev-parse --verify FETCH_HEAD)" + + VALIDATE_COMMIT_LOG="$VALIDATE_UPSTREAM..$VALIDATE_HEAD" + VALIDATE_COMMIT_DIFF="$VALIDATE_UPSTREAM...$VALIDATE_HEAD" + + validate_diff() { + if [ "$VALIDATE_UPSTREAM" != "$VALIDATE_HEAD" ]; then + git diff "$VALIDATE_COMMIT_DIFF" "$@" + fi + } + validate_log() { + if [ "$VALIDATE_UPSTREAM" != "$VALIDATE_HEAD" ]; then + git log "$VALIDATE_COMMIT_LOG" "$@" + fi + } +fi diff --git a/Godeps/_workspace/src/github.com/docker/libkv/script/coverage b/Godeps/_workspace/src/github.com/docker/libkv/script/coverage new file mode 100644 index 000000000..a7a13f450 --- /dev/null +++ b/Godeps/_workspace/src/github.com/docker/libkv/script/coverage @@ -0,0 +1,21 @@ +#!/bin/bash + +MODE="mode: count" +ROOT=${TRAVIS_BUILD_DIR:-.}/../../.. + +# Grab the list of packages. +# Exclude the API and CLI from coverage as it will be covered by integration tests. +PACKAGES=`go list ./...` + +# Create the empty coverage file. +echo $MODE > goverage.report + +# Run coverage on every package. +for package in $PACKAGES; do + output="$ROOT/$package/coverage.out" + + go test -test.short -covermode=count -coverprofile=$output $package + if [ -f "$output" ] ; then + cat "$output" | grep -v "$MODE" >> goverage.report + fi +done diff --git a/Godeps/_workspace/src/github.com/docker/libkv/script/travis_consul.sh b/Godeps/_workspace/src/github.com/docker/libkv/script/travis_consul.sh new file mode 100644 index 000000000..389971163 --- /dev/null +++ b/Godeps/_workspace/src/github.com/docker/libkv/script/travis_consul.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +if [ $# -gt 0 ] ; then + CONSUL_VERSION="$1" +else + CONSUL_VERSION="0.5.2" +fi + +# install consul +wget "https://dl.bintray.com/mitchellh/consul/${CONSUL_VERSION}_linux_amd64.zip" +unzip "${CONSUL_VERSION}_linux_amd64.zip" + +# make config for minimum ttl +touch config.json +echo "{\"session_ttl_min\": \"1s\"}" >> config.json + +# check +./consul --version diff --git a/Godeps/_workspace/src/github.com/docker/libkv/script/travis_etcd.sh b/Godeps/_workspace/src/github.com/docker/libkv/script/travis_etcd.sh new file mode 100644 index 000000000..004154969 --- /dev/null +++ b/Godeps/_workspace/src/github.com/docker/libkv/script/travis_etcd.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +if [ $# -gt 0 ] ; then + ETCD_VERSION="$1" +else + ETCD_VERSION="2.0.11" +fi + +curl -L https://github.com/coreos/etcd/releases/download/v$ETCD_VERSION/etcd-v$ETCD_VERSION-linux-amd64.tar.gz -o etcd-v$ETCD_VERSION-linux-amd64.tar.gz +tar xzvf etcd-v$ETCD_VERSION-linux-amd64.tar.gz +mv etcd-v$ETCD_VERSION-linux-amd64 etcd diff --git a/Godeps/_workspace/src/github.com/docker/libkv/script/travis_zk.sh b/Godeps/_workspace/src/github.com/docker/libkv/script/travis_zk.sh new file mode 100644 index 000000000..cc4dc0556 --- /dev/null +++ b/Godeps/_workspace/src/github.com/docker/libkv/script/travis_zk.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +if [ $# -gt 0 ] ; then + ZK_VERSION="$1" +else + ZK_VERSION="3.4.6" +fi + +wget "http://mirrors.ukfast.co.uk/sites/ftp.apache.org/zookeeper/stable/zookeeper-${ZK_VERSION}.tar.gz" +tar -xvf "zookeeper-${ZK_VERSION}.tar.gz" +mv zookeeper-$ZK_VERSION zk +mv ./zk/conf/zoo_sample.cfg ./zk/conf/zoo.cfg diff --git a/Godeps/_workspace/src/github.com/docker/libkv/script/validate-gofmt b/Godeps/_workspace/src/github.com/docker/libkv/script/validate-gofmt new file mode 100644 index 000000000..c565976b4 --- /dev/null +++ b/Godeps/_workspace/src/github.com/docker/libkv/script/validate-gofmt @@ -0,0 +1,30 @@ +#!/bin/bash + +source "$(dirname "$BASH_SOURCE")/.validate" + +IFS=$'\n' +files=( $(validate_diff --diff-filter=ACMR --name-only -- '*.go' | grep -v '^Godeps/' || true) ) +unset IFS + +badFiles=() +for f in "${files[@]}"; do + # we use "git show" here to validate that what's committed is formatted + if [ "$(git show "$VALIDATE_HEAD:$f" | gofmt -s -l)" ]; then + badFiles+=( "$f" ) + fi +done + +if [ ${#badFiles[@]} -eq 0 ]; then + echo 'Congratulations! All Go source files are properly formatted.' +else + { + echo "These files are not properly gofmt'd:" + for f in "${badFiles[@]}"; do + echo " - $f" + done + echo + echo 'Please reformat the above files using "gofmt -s -w" and commit the result.' + echo + } >&2 + false +fi diff --git a/Godeps/_workspace/src/github.com/docker/libkv/store/consul/consul.go b/Godeps/_workspace/src/github.com/docker/libkv/store/consul/consul.go new file mode 100644 index 000000000..ebabde5ff --- /dev/null +++ b/Godeps/_workspace/src/github.com/docker/libkv/store/consul/consul.go @@ -0,0 +1,422 @@ +package consul + +import ( + "crypto/tls" + "net/http" + "strings" + "sync" + "time" + + "github.com/docker/libkv/store" + api "github.com/hashicorp/consul/api" +) + +const ( + // DefaultWatchWaitTime is how long we block for at a + // time to check if the watched key has changed. This + // affects the minimum time it takes to cancel a watch. + DefaultWatchWaitTime = 15 * time.Second +) + +// Consul is the receiver type for the +// Store interface +type Consul struct { + sync.Mutex + config *api.Config + client *api.Client + ephemeralTTL time.Duration +} + +type consulLock struct { + lock *api.Lock +} + +// New creates a new Consul client given a list +// of endpoints and optional tls config +func New(endpoints []string, options *store.Config) (store.Store, error) { + s := &Consul{} + + // Create Consul client + config := api.DefaultConfig() + s.config = config + config.HttpClient = http.DefaultClient + config.Address = endpoints[0] + config.Scheme = "http" + + // Set options + if options != nil { + if options.TLS != nil { + s.setTLS(options.TLS) + } + if options.ConnectionTimeout != 0 { + s.setTimeout(options.ConnectionTimeout) + } + if options.EphemeralTTL != 0 { + s.setEphemeralTTL(options.EphemeralTTL) + } + } + + // Creates a new client + client, err := api.NewClient(config) + if err != nil { + return nil, err + } + s.client = client + + return s, nil +} + +// SetTLS sets Consul TLS options +func (s *Consul) setTLS(tls *tls.Config) { + s.config.HttpClient.Transport = &http.Transport{ + TLSClientConfig: tls, + } + s.config.Scheme = "https" +} + +// SetTimeout sets the timout for connecting to Consul +func (s *Consul) setTimeout(time time.Duration) { + s.config.WaitTime = time +} + +// SetEphemeralTTL sets the ttl for ephemeral nodes +func (s *Consul) setEphemeralTTL(ttl time.Duration) { + s.ephemeralTTL = ttl +} + +// Normalize the key for usage in Consul +func (s *Consul) normalize(key string) string { + key = store.Normalize(key) + return strings.TrimPrefix(key, "/") +} + +func (s *Consul) refreshSession(pair *api.KVPair) error { + // Check if there is any previous session with an active TTL + session, err := s.getActiveSession(pair.Key) + if err != nil { + return err + } + + if session == "" { + entry := &api.SessionEntry{ + Behavior: api.SessionBehaviorDelete, // Delete the key when the session expires + TTL: ((s.ephemeralTTL) / 2).String(), // Consul multiplies the TTL by 2x + LockDelay: 1 * time.Millisecond, // Virtually disable lock delay + } + + // Create the key session + session, _, err = s.client.Session().Create(entry, nil) + if err != nil { + return err + } + + lockOpts := &api.LockOptions{ + Key: pair.Key, + Session: session, + } + + // Lock and ignore if lock is held + // It's just a placeholder for the + // ephemeral behavior + lock, _ := s.client.LockOpts(lockOpts) + if lock != nil { + lock.Lock(nil) + } + } + + _, _, err = s.client.Session().Renew(session, nil) + if err != nil { + return s.refreshSession(pair) + } + return nil +} + +// getActiveSession checks if the key already has +// a session attached +func (s *Consul) getActiveSession(key string) (string, error) { + pair, _, err := s.client.KV().Get(key, nil) + if err != nil { + return "", err + } + if pair != nil && pair.Session != "" { + return pair.Session, nil + } + return "", nil +} + +// Get the value at "key", returns the last modified index +// to use in conjunction to CAS calls +func (s *Consul) Get(key string) (*store.KVPair, error) { + options := &api.QueryOptions{ + AllowStale: false, + RequireConsistent: true, + } + + pair, meta, err := s.client.KV().Get(s.normalize(key), options) + if err != nil { + return nil, err + } + + // If pair is nil then the key does not exist + if pair == nil { + return nil, store.ErrKeyNotFound + } + + return &store.KVPair{Key: pair.Key, Value: pair.Value, LastIndex: meta.LastIndex}, nil +} + +// Put a value at "key" +func (s *Consul) Put(key string, value []byte, opts *store.WriteOptions) error { + key = s.normalize(key) + + p := &api.KVPair{ + Key: key, + Value: value, + } + + if opts != nil && opts.Ephemeral { + // Create or refresh the session + err := s.refreshSession(p) + if err != nil { + return err + } + } + + _, err := s.client.KV().Put(p, nil) + return err +} + +// Delete a value at "key" +func (s *Consul) Delete(key string) error { + _, err := s.client.KV().Delete(s.normalize(key), nil) + return err +} + +// Exists checks that the key exists inside the store +func (s *Consul) Exists(key string) (bool, error) { + _, err := s.Get(key) + if err != nil && err == store.ErrKeyNotFound { + return false, err + } + return true, nil +} + +// List child nodes of a given directory +func (s *Consul) List(directory string) ([]*store.KVPair, error) { + pairs, _, err := s.client.KV().List(s.normalize(directory), nil) + if err != nil { + return nil, err + } + if len(pairs) == 0 { + return nil, store.ErrKeyNotFound + } + + kv := []*store.KVPair{} + + for _, pair := range pairs { + if pair.Key == directory { + continue + } + kv = append(kv, &store.KVPair{ + Key: pair.Key, + Value: pair.Value, + LastIndex: pair.ModifyIndex, + }) + } + + return kv, nil +} + +// DeleteTree deletes a range of keys under a given directory +func (s *Consul) DeleteTree(directory string) error { + _, err := s.client.KV().DeleteTree(s.normalize(directory), nil) + return err +} + +// Watch for changes on a "key" +// It returns a channel that will receive changes or pass +// on errors. Upon creation, the current value will first +// be sent to the channel. Providing a non-nil stopCh can +// be used to stop watching. +func (s *Consul) Watch(key string, stopCh <-chan struct{}) (<-chan *store.KVPair, error) { + kv := s.client.KV() + watchCh := make(chan *store.KVPair) + + go func() { + defer close(watchCh) + + // Use a wait time in order to check if we should quit + // from time to time. + opts := &api.QueryOptions{WaitTime: DefaultWatchWaitTime} + + for { + // Check if we should quit + select { + case <-stopCh: + return + default: + } + + // Get the key + pair, meta, err := kv.Get(key, opts) + if err != nil { + return + } + + // If LastIndex didn't change then it means `Get` returned + // because of the WaitTime and the key didn't changed. + if opts.WaitIndex == meta.LastIndex { + continue + } + opts.WaitIndex = meta.LastIndex + + // Return the value to the channel + // FIXME: What happens when a key is deleted? + if pair != nil { + watchCh <- &store.KVPair{ + Key: pair.Key, + Value: pair.Value, + LastIndex: pair.ModifyIndex, + } + } + } + }() + + return watchCh, nil +} + +// WatchTree watches for changes on a "directory" +// It returns a channel that will receive changes or pass +// on errors. Upon creating a watch, the current childs values +// will be sent to the channel .Providing a non-nil stopCh can +// be used to stop watching. +func (s *Consul) WatchTree(directory string, stopCh <-chan struct{}) (<-chan []*store.KVPair, error) { + kv := s.client.KV() + watchCh := make(chan []*store.KVPair) + + go func() { + defer close(watchCh) + + // Use a wait time in order to check if we should quit + // from time to time. + opts := &api.QueryOptions{WaitTime: DefaultWatchWaitTime} + for { + // Check if we should quit + select { + case <-stopCh: + return + default: + } + + // Get all the childrens + pairs, meta, err := kv.List(directory, opts) + if err != nil { + return + } + + // If LastIndex didn't change then it means `Get` returned + // because of the WaitTime and the child keys didn't change. + if opts.WaitIndex == meta.LastIndex { + continue + } + opts.WaitIndex = meta.LastIndex + + // Return children KV pairs to the channel + kvpairs := []*store.KVPair{} + for _, pair := range pairs { + if pair.Key == directory { + continue + } + kvpairs = append(kvpairs, &store.KVPair{ + Key: pair.Key, + Value: pair.Value, + LastIndex: pair.ModifyIndex, + }) + } + watchCh <- kvpairs + } + }() + + return watchCh, nil +} + +// NewLock returns a handle to a lock struct which can +// be used to provide mutual exclusion on a key +func (s *Consul) NewLock(key string, options *store.LockOptions) (store.Locker, error) { + consulOpts := &api.LockOptions{ + Key: s.normalize(key), + } + + if options != nil { + consulOpts.Value = options.Value + } + + l, err := s.client.LockOpts(consulOpts) + if err != nil { + return nil, err + } + + return &consulLock{lock: l}, nil +} + +// Lock attempts to acquire the lock and blocks while +// doing so. It returns a channel that is closed if our +// lock is lost or if an error occurs +func (l *consulLock) Lock() (<-chan struct{}, error) { + return l.lock.Lock(nil) +} + +// Unlock the "key". Calling unlock while +// not holding the lock will throw an error +func (l *consulLock) Unlock() error { + return l.lock.Unlock() +} + +// AtomicPut put a value at "key" if the key has not been +// modified in the meantime, throws an error if this is the case +func (s *Consul) AtomicPut(key string, value []byte, previous *store.KVPair, options *store.WriteOptions) (bool, *store.KVPair, error) { + + p := &api.KVPair{Key: s.normalize(key), Value: value} + + if previous == nil { + // Consul interprets ModifyIndex = 0 as new key. + p.ModifyIndex = 0 + } else { + p.ModifyIndex = previous.LastIndex + } + + if work, _, err := s.client.KV().CAS(p, nil); err != nil { + return false, nil, err + } else if !work { + return false, nil, store.ErrKeyModified + } + + pair, err := s.Get(key) + if err != nil { + return false, nil, err + } + + return true, pair, nil +} + +// AtomicDelete deletes a value at "key" if the key has not +// been modified in the meantime, throws an error if this is the case +func (s *Consul) AtomicDelete(key string, previous *store.KVPair) (bool, error) { + if previous == nil { + return false, store.ErrPreviousNotSpecified + } + + p := &api.KVPair{Key: s.normalize(key), ModifyIndex: previous.LastIndex} + if work, _, err := s.client.KV().DeleteCAS(p, nil); err != nil { + return false, err + } else if !work { + return false, store.ErrKeyModified + } + + return true, nil +} + +// Close closes the client connection +func (s *Consul) Close() { + return +} diff --git a/Godeps/_workspace/src/github.com/docker/libkv/store/consul/consul_test.go b/Godeps/_workspace/src/github.com/docker/libkv/store/consul/consul_test.go new file mode 100644 index 000000000..26e20bb47 --- /dev/null +++ b/Godeps/_workspace/src/github.com/docker/libkv/store/consul/consul_test.go @@ -0,0 +1,62 @@ +package consul + +import ( + "testing" + "time" + + "github.com/docker/libkv/store" + "github.com/docker/libkv/testutils" + "github.com/stretchr/testify/assert" +) + +func makeConsulClient(t *testing.T) store.Store { + client := "localhost:8500" + + kv, err := New( + []string{client}, + &store.Config{ + ConnectionTimeout: 3 * time.Second, + EphemeralTTL: 2 * time.Second, + }, + ) + + if err != nil { + t.Fatalf("cannot create store: %v", err) + } + + return kv +} + +func TestConsulStore(t *testing.T) { + kv := makeConsulClient(t) + backup := makeConsulClient(t) + + testutils.RunTestStore(t, kv, backup) +} + +func TestGetActiveSession(t *testing.T) { + kv := makeConsulClient(t) + + consul := kv.(*Consul) + + key := "foo" + value := []byte("bar") + + // Put the first key with the Ephemeral flag + err := kv.Put(key, value, &store.WriteOptions{Ephemeral: true}) + assert.NoError(t, err) + + // Session should not be empty + session, err := consul.getActiveSession(key) + assert.NoError(t, err) + assert.NotEqual(t, session, "") + + // Delete the key + err = kv.Delete(key) + assert.NoError(t, err) + + // Check the session again, it should return nothing + session, err = consul.getActiveSession(key) + assert.NoError(t, err) + assert.Equal(t, session, "") +} diff --git a/Godeps/_workspace/src/github.com/docker/libkv/store/etcd/etcd.go b/Godeps/_workspace/src/github.com/docker/libkv/store/etcd/etcd.go new file mode 100644 index 000000000..30d2e0831 --- /dev/null +++ b/Godeps/_workspace/src/github.com/docker/libkv/store/etcd/etcd.go @@ -0,0 +1,499 @@ +package etcd + +import ( + "crypto/tls" + "net" + "net/http" + "strings" + "time" + + etcd "github.com/coreos/go-etcd/etcd" + "github.com/docker/libkv/store" +) + +// Etcd is the receiver type for the +// Store interface +type Etcd struct { + client *etcd.Client + ephemeralTTL time.Duration +} + +type etcdLock struct { + client *etcd.Client + stopLock chan struct{} + key string + value string + last *etcd.Response + ttl uint64 +} + +const ( + periodicSync = 10 * time.Minute + defaultLockTTL = 20 * time.Second + defaultUpdateTime = 5 * time.Second +) + +// New creates a new Etcd client given a list +// of endpoints and an optional tls config +func New(addrs []string, options *store.Config) (store.Store, error) { + s := &Etcd{} + + entries := store.CreateEndpoints(addrs, "http") + s.client = etcd.NewClient(entries) + + // Set options + if options != nil { + if options.TLS != nil { + s.setTLS(options.TLS) + } + if options.ConnectionTimeout != 0 { + s.setTimeout(options.ConnectionTimeout) + } + if options.EphemeralTTL != 0 { + s.setEphemeralTTL(options.EphemeralTTL) + } + } + + // Periodic SyncCluster + go func() { + for { + s.client.SyncCluster() + time.Sleep(periodicSync) + } + }() + + return s, nil +} + +// SetTLS sets the tls configuration given the path +// of certificate files +func (s *Etcd) setTLS(tls *tls.Config) { + // Change to https scheme + var addrs []string + entries := s.client.GetCluster() + for _, entry := range entries { + addrs = append(addrs, strings.Replace(entry, "http", "https", -1)) + } + s.client.SetCluster(addrs) + + // Set transport + t := http.Transport{ + Dial: (&net.Dialer{ + Timeout: 30 * time.Second, + KeepAlive: 30 * time.Second, + }).Dial, + TLSHandshakeTimeout: 10 * time.Second, + TLSClientConfig: tls, + } + s.client.SetTransport(&t) +} + +// setTimeout sets the timeout used for connecting to the store +func (s *Etcd) setTimeout(time time.Duration) { + s.client.SetDialTimeout(time) +} + +// setEphemeralHeartbeat sets the heartbeat value to notify +// that a node is alive +func (s *Etcd) setEphemeralTTL(time time.Duration) { + s.ephemeralTTL = time +} + +// createDirectory creates the entire path for a directory +// that does not exist +func (s *Etcd) createDirectory(path string) error { + if _, err := s.client.CreateDir(store.Normalize(path), 10); err != nil { + if etcdError, ok := err.(*etcd.EtcdError); ok { + // Skip key already exists + if etcdError.ErrorCode != 105 { + return err + } + } else { + return err + } + } + return nil +} + +// Get the value at "key", returns the last modified index +// to use in conjunction to Atomic calls +func (s *Etcd) Get(key string) (pair *store.KVPair, err error) { + result, err := s.client.Get(store.Normalize(key), false, false) + if err != nil { + if etcdError, ok := err.(*etcd.EtcdError); ok { + // Not a Directory or Not a file + if etcdError.ErrorCode == 102 || etcdError.ErrorCode == 104 { + return nil, store.ErrKeyNotFound + } + } + return nil, err + } + + pair = &store.KVPair{ + Key: key, + Value: []byte(result.Node.Value), + LastIndex: result.Node.ModifiedIndex, + } + + return pair, nil +} + +// Put a value at "key" +func (s *Etcd) Put(key string, value []byte, opts *store.WriteOptions) error { + + // Default TTL = 0 means no expiration + var ttl uint64 + if opts != nil && opts.Ephemeral { + ttl = uint64(s.ephemeralTTL.Seconds()) + } + + if _, err := s.client.Set(key, string(value), ttl); err != nil { + if etcdError, ok := err.(*etcd.EtcdError); ok { + + // Not a directory + if etcdError.ErrorCode == 104 { + // Remove the last element (the actual key) + // and create the full directory path + err = s.createDirectory(store.GetDirectory(key)) + if err != nil { + return err + } + + // Now that the directory is created, set the key + if _, err := s.client.Set(key, string(value), ttl); err != nil { + return err + } + } + } + return err + } + return nil +} + +// Delete a value at "key" +func (s *Etcd) Delete(key string) error { + _, err := s.client.Delete(store.Normalize(key), false) + return err +} + +// Exists checks if the key exists inside the store +func (s *Etcd) Exists(key string) (bool, error) { + entry, err := s.Get(key) + if err != nil && entry != nil { + if err == store.ErrKeyNotFound || entry.Value == nil { + return false, nil + } + return false, err + } + return true, nil +} + +// Watch for changes on a "key" +// It returns a channel that will receive changes or pass +// on errors. Upon creation, the current value will first +// be sent to the channel. Providing a non-nil stopCh can +// be used to stop watching. +func (s *Etcd) Watch(key string, stopCh <-chan struct{}) (<-chan *store.KVPair, error) { + // Get the current value + current, err := s.Get(key) + if err != nil { + return nil, err + } + + // Start an etcd watch. + // Note: etcd will send the current value through the channel. + etcdWatchCh := make(chan *etcd.Response) + etcdStopCh := make(chan bool) + go s.client.Watch(store.Normalize(key), 0, false, etcdWatchCh, etcdStopCh) + + // Adapter goroutine: The goal here is to convert whatever + // format etcd is using into our interface. + watchCh := make(chan *store.KVPair) + go func() { + defer close(watchCh) + + // Push the current value through the channel. + watchCh <- current + + for { + select { + case result := <-etcdWatchCh: + watchCh <- &store.KVPair{ + Key: key, + Value: []byte(result.Node.Value), + LastIndex: result.Node.ModifiedIndex, + } + case <-stopCh: + etcdStopCh <- true + return + } + } + }() + return watchCh, nil +} + +// WatchTree watches for changes on a "directory" +// It returns a channel that will receive changes or pass +// on errors. Upon creating a watch, the current childs values +// will be sent to the channel .Providing a non-nil stopCh can +// be used to stop watching. +func (s *Etcd) WatchTree(directory string, stopCh <-chan struct{}) (<-chan []*store.KVPair, error) { + // Get child values + current, err := s.List(directory) + if err != nil { + return nil, err + } + + // Start the watch + etcdWatchCh := make(chan *etcd.Response) + etcdStopCh := make(chan bool) + go s.client.Watch(store.Normalize(directory), 0, true, etcdWatchCh, etcdStopCh) + + // Adapter goroutine: The goal here is to convert whatever + // format etcd is using into our interface. + watchCh := make(chan []*store.KVPair) + go func() { + defer close(watchCh) + + // Push the current value through the channel. + watchCh <- current + + for { + select { + case <-etcdWatchCh: + // FIXME: We should probably use the value pushed by the channel. + // However, Node.Nodes seems to be empty. + if list, err := s.List(directory); err == nil { + watchCh <- list + } + case <-stopCh: + etcdStopCh <- true + return + } + } + }() + return watchCh, nil +} + +// AtomicPut put a value at "key" if the key has not been +// modified in the meantime, throws an error if this is the case +func (s *Etcd) AtomicPut(key string, value []byte, previous *store.KVPair, options *store.WriteOptions) (bool, *store.KVPair, error) { + + var meta *etcd.Response + var err error + if previous != nil { + meta, err = s.client.CompareAndSwap(store.Normalize(key), string(value), 0, "", previous.LastIndex) + } else { + // Interpret previous == nil as Atomic Create + meta, err = s.client.Create(store.Normalize(key), string(value), 0) + if etcdError, ok := err.(*etcd.EtcdError); ok { + + // Directory doesn't exist. + if etcdError.ErrorCode == 104 { + // Remove the last element (the actual key) + // and create the full directory path + err = s.createDirectory(store.GetDirectory(key)) + if err != nil { + return false, nil, err + } + + // Now that the directory is created, create the key + if _, err := s.client.Create(key, string(value), 0); err != nil { + return false, nil, err + } + } + } + } + if err != nil { + if etcdError, ok := err.(*etcd.EtcdError); ok { + // Compare Failed + if etcdError.ErrorCode == 101 { + return false, nil, store.ErrKeyModified + } + } + return false, nil, err + } + + updated := &store.KVPair{ + Key: key, + Value: value, + LastIndex: meta.Node.ModifiedIndex, + } + + return true, updated, nil +} + +// AtomicDelete deletes a value at "key" if the key +// has not been modified in the meantime, throws an +// error if this is the case +func (s *Etcd) AtomicDelete(key string, previous *store.KVPair) (bool, error) { + if previous == nil { + return false, store.ErrPreviousNotSpecified + } + + _, err := s.client.CompareAndDelete(store.Normalize(key), "", previous.LastIndex) + if err != nil { + if etcdError, ok := err.(*etcd.EtcdError); ok { + // Compare failed + if etcdError.ErrorCode == 101 { + return false, store.ErrKeyModified + } + } + return false, err + } + + return true, nil +} + +// List child nodes of a given directory +func (s *Etcd) List(directory string) ([]*store.KVPair, error) { + resp, err := s.client.Get(store.Normalize(directory), true, true) + if err != nil { + return nil, err + } + kv := []*store.KVPair{} + for _, n := range resp.Node.Nodes { + key := strings.TrimLeft(n.Key, "/") + kv = append(kv, &store.KVPair{ + Key: key, + Value: []byte(n.Value), + LastIndex: n.ModifiedIndex, + }) + } + return kv, nil +} + +// DeleteTree deletes a range of keys under a given directory +func (s *Etcd) DeleteTree(directory string) error { + _, err := s.client.Delete(store.Normalize(directory), true) + return err +} + +// NewLock returns a handle to a lock struct which can +// be used to provide mutual exclusion on a key +func (s *Etcd) NewLock(key string, options *store.LockOptions) (lock store.Locker, err error) { + var value string + ttl := uint64(time.Duration(defaultLockTTL).Seconds()) + + // Apply options on Lock + if options != nil { + if options.Value != nil { + value = string(options.Value) + } + if options.TTL != 0 { + ttl = uint64(options.TTL.Seconds()) + } + } + + // Create lock object + lock = &etcdLock{ + client: s.client, + key: key, + value: value, + ttl: ttl, + } + + return lock, nil +} + +// Lock attempts to acquire the lock and blocks while +// doing so. It returns a channel that is closed if our +// lock is lost or if an error occurs +func (l *etcdLock) Lock() (<-chan struct{}, error) { + + key := store.Normalize(l.key) + + // Lock holder channels + lockHeld := make(chan struct{}) + stopLocking := make(chan struct{}) + + var lastIndex uint64 + + for { + resp, err := l.client.Create(key, l.value, l.ttl) + if err != nil { + if etcdError, ok := err.(*etcd.EtcdError); ok { + // Key already exists + if etcdError.ErrorCode != 105 { + lastIndex = ^uint64(0) + } + } + } else { + lastIndex = resp.Node.ModifiedIndex + } + + _, err = l.client.CompareAndSwap(key, l.value, l.ttl, "", lastIndex) + + if err == nil { + // Leader section + l.stopLock = stopLocking + go l.holdLock(key, lockHeld, stopLocking) + break + } else { + // Seeker section + chW := make(chan *etcd.Response) + chWStop := make(chan bool) + l.waitLock(key, chW, chWStop) + + // Delete or Expire event occured + // Retry + } + } + + return lockHeld, nil +} + +// Hold the lock as long as we can +// Updates the key ttl periodically until we receive +// an explicit stop signal from the Unlock method +func (l *etcdLock) holdLock(key string, lockHeld chan struct{}, stopLocking chan struct{}) { + defer close(lockHeld) + + update := time.NewTicker(defaultUpdateTime) + defer update.Stop() + + var err error + + for { + select { + case <-update.C: + l.last, err = l.client.Update(key, l.value, l.ttl) + if err != nil { + return + } + + case <-stopLocking: + return + } + } +} + +// WaitLock simply waits for the key to be available for creation +func (l *etcdLock) waitLock(key string, eventCh chan *etcd.Response, stopWatchCh chan bool) { + go l.client.Watch(key, 0, false, eventCh, stopWatchCh) + for event := range eventCh { + if event.Action == "delete" || event.Action == "expire" { + return + } + } +} + +// Unlock the "key". Calling unlock while +// not holding the lock will throw an error +func (l *etcdLock) Unlock() error { + if l.stopLock != nil { + l.stopLock <- struct{}{} + } + if l.last != nil { + _, err := l.client.CompareAndDelete(store.Normalize(l.key), l.value, l.last.Node.ModifiedIndex) + if err != nil { + return err + } + } + return nil +} + +// Close closes the client connection +func (s *Etcd) Close() { + return +} diff --git a/Godeps/_workspace/src/github.com/docker/libkv/store/etcd/etcd_test.go b/Godeps/_workspace/src/github.com/docker/libkv/store/etcd/etcd_test.go new file mode 100644 index 000000000..73d1b06f9 --- /dev/null +++ b/Godeps/_workspace/src/github.com/docker/libkv/store/etcd/etcd_test.go @@ -0,0 +1,34 @@ +package etcd + +import ( + "testing" + "time" + + "github.com/docker/libkv/store" + "github.com/docker/libkv/testutils" +) + +func makeEtcdClient(t *testing.T) store.Store { + client := "localhost:4001" + + kv, err := New( + []string{client}, + &store.Config{ + ConnectionTimeout: 3 * time.Second, + EphemeralTTL: 2 * time.Second, + }, + ) + + if err != nil { + t.Fatalf("cannot create store: %v", err) + } + + return kv +} + +func TestEtcdStore(t *testing.T) { + kv := makeEtcdClient(t) + backup := makeEtcdClient(t) + + testutils.RunTestStore(t, kv, backup) +} diff --git a/Godeps/_workspace/src/github.com/docker/libkv/store/helpers.go b/Godeps/_workspace/src/github.com/docker/libkv/store/helpers.go new file mode 100644 index 000000000..0fb74c9ae --- /dev/null +++ b/Godeps/_workspace/src/github.com/docker/libkv/store/helpers.go @@ -0,0 +1,47 @@ +package store + +import ( + "strings" +) + +// CreateEndpoints creates a list of endpoints given the right scheme +func CreateEndpoints(addrs []string, scheme string) (entries []string) { + for _, addr := range addrs { + entries = append(entries, scheme+"://"+addr) + } + return entries +} + +// Normalize the key for each store to the form: +// +// /path/to/key +// +func Normalize(key string) string { + return "/" + join(SplitKey(key)) +} + +// GetDirectory gets the full directory part of +// the key to the form: +// +// /path/to/ +// +func GetDirectory(key string) string { + parts := SplitKey(key) + parts = parts[:len(parts)-1] + return "/" + join(parts) +} + +// SplitKey splits the key to extract path informations +func SplitKey(key string) (path []string) { + if strings.Contains(key, "/") { + path = strings.Split(key, "/") + } else { + path = []string{key} + } + return path +} + +// join the path parts with '/' +func join(parts []string) string { + return strings.Join(parts, "/") +} diff --git a/Godeps/_workspace/src/github.com/docker/libkv/store/mock/mock.go b/Godeps/_workspace/src/github.com/docker/libkv/store/mock/mock.go new file mode 100644 index 000000000..9f63e50ae --- /dev/null +++ b/Godeps/_workspace/src/github.com/docker/libkv/store/mock/mock.go @@ -0,0 +1,113 @@ +package mock + +import ( + "github.com/docker/libkv/store" + "github.com/stretchr/testify/mock" +) + +// Mock store. Mocks all Store functions using testify.Mock +type Mock struct { + mock.Mock + + // Endpoints passed to InitializeMock + Endpoints []string + + // Options passed to InitializeMock + Options *store.Config +} + +// New creates a Mock store +func New(endpoints []string, options *store.Config) (store.Store, error) { + s := &Mock{} + s.Endpoints = endpoints + s.Options = options + return s, nil +} + +// Put mock +func (s *Mock) Put(key string, value []byte, opts *store.WriteOptions) error { + args := s.Mock.Called(key, value, opts) + return args.Error(0) +} + +// Get mock +func (s *Mock) Get(key string) (*store.KVPair, error) { + args := s.Mock.Called(key) + return args.Get(0).(*store.KVPair), args.Error(1) +} + +// Delete mock +func (s *Mock) Delete(key string) error { + args := s.Mock.Called(key) + return args.Error(0) +} + +// Exists mock +func (s *Mock) Exists(key string) (bool, error) { + args := s.Mock.Called(key) + return args.Bool(0), args.Error(1) +} + +// Watch mock +func (s *Mock) Watch(key string, stopCh <-chan struct{}) (<-chan *store.KVPair, error) { + args := s.Mock.Called(key, stopCh) + return args.Get(0).(<-chan *store.KVPair), args.Error(1) +} + +// WatchTree mock +func (s *Mock) WatchTree(prefix string, stopCh <-chan struct{}) (<-chan []*store.KVPair, error) { + args := s.Mock.Called(prefix, stopCh) + return args.Get(0).(chan []*store.KVPair), args.Error(1) +} + +// NewLock mock +func (s *Mock) NewLock(key string, options *store.LockOptions) (store.Locker, error) { + args := s.Mock.Called(key, options) + return args.Get(0).(store.Locker), args.Error(1) +} + +// List mock +func (s *Mock) List(prefix string) ([]*store.KVPair, error) { + args := s.Mock.Called(prefix) + return args.Get(0).([]*store.KVPair), args.Error(1) +} + +// DeleteTree mock +func (s *Mock) DeleteTree(prefix string) error { + args := s.Mock.Called(prefix) + return args.Error(0) +} + +// AtomicPut mock +func (s *Mock) AtomicPut(key string, value []byte, previous *store.KVPair, opts *store.WriteOptions) (bool, *store.KVPair, error) { + args := s.Mock.Called(key, value, previous, opts) + return args.Bool(0), args.Get(1).(*store.KVPair), args.Error(2) +} + +// AtomicDelete mock +func (s *Mock) AtomicDelete(key string, previous *store.KVPair) (bool, error) { + args := s.Mock.Called(key, previous) + return args.Bool(0), args.Error(1) +} + +// Lock mock implementation of Locker +type Lock struct { + mock.Mock +} + +// Lock mock +func (l *Lock) Lock() (<-chan struct{}, error) { + args := l.Mock.Called() + return args.Get(0).(<-chan struct{}), args.Error(1) +} + +// Unlock mock +func (l *Lock) Unlock() error { + args := l.Mock.Called() + return args.Error(0) +} + +// Close mock +func (s *Mock) Close() { + return +} diff --git a/Godeps/_workspace/src/github.com/docker/libkv/store/store.go b/Godeps/_workspace/src/github.com/docker/libkv/store/store.go new file mode 100644 index 000000000..49e2eb9dc --- /dev/null +++ b/Godeps/_workspace/src/github.com/docker/libkv/store/store.go @@ -0,0 +1,119 @@ +package store + +import ( + "crypto/tls" + "errors" + "time" +) + +// Backend represents a KV Store Backend +type Backend string + +const ( + // CONSUL backend + CONSUL Backend = "consul" + // ETCD backend + ETCD Backend = "etcd" + // ZK backend + ZK Backend = "zk" +) + +var ( + // ErrNotSupported is thrown when the backend k/v store is not supported by libkv + ErrNotSupported = errors.New("Backend storage not supported yet, please choose another one") + // ErrNotImplemented is thrown when a method is not implemented by the current backend + ErrNotImplemented = errors.New("Call not implemented in current backend") + // ErrNotReachable is thrown when the API cannot be reached for issuing common store operations + ErrNotReachable = errors.New("Api not reachable") + // ErrCannotLock is thrown when there is an error acquiring a lock on a key + ErrCannotLock = errors.New("Error acquiring the lock") + // ErrKeyModified is thrown during an atomic operation if the index does not match the one in the store + ErrKeyModified = errors.New("Unable to complete atomic operation, key modified") + // ErrKeyNotFound is thrown when the key is not found in the store during a Get operation + ErrKeyNotFound = errors.New("Key not found in store") + // ErrPreviousNotSpecified is thrown when the previous value is not specified for an atomic operation + ErrPreviousNotSpecified = errors.New("Previous K/V pair should be provided for the Atomic operation") +) + +// Config contains the options for a storage client +type Config struct { + TLS *tls.Config + ConnectionTimeout time.Duration + EphemeralTTL time.Duration +} + +// Store represents the backend K/V storage +// Each store should support every call listed +// here. Or it couldn't be implemented as a K/V +// backend for libkv +type Store interface { + // Put a value at the specified key + Put(key string, value []byte, options *WriteOptions) error + + // Get a value given its key + Get(key string) (*KVPair, error) + + // Delete the value at the specified key + Delete(key string) error + + // Verify if a Key exists in the store + Exists(key string) (bool, error) + + // Watch for changes on a key + Watch(key string, stopCh <-chan struct{}) (<-chan *KVPair, error) + + // WatchTree watches for changes on child nodes under + // a given a directory + WatchTree(directory string, stopCh <-chan struct{}) (<-chan []*KVPair, error) + + // CreateLock for a given key. + // The returned Locker is not held and must be acquired + // with `.Lock`. The Value is optional. + NewLock(key string, options *LockOptions) (Locker, error) + + // List the content of a given prefix + List(directory string) ([]*KVPair, error) + + // DeleteTree deletes a range of keys under a given directory + DeleteTree(directory string) error + + // Atomic CAS operation on a single value. + // Pass previous = nil to create a new key. + AtomicPut(key string, value []byte, previous *KVPair, options *WriteOptions) (bool, *KVPair, error) + + // Atomic delete of a single value + AtomicDelete(key string, previous *KVPair) (bool, error) + + // Close the store connection + Close() +} + +// KVPair represents {Key, Value, Lastindex} tuple +type KVPair struct { + Key string + Value []byte + LastIndex uint64 +} + +// WriteOptions contains optional request parameters +type WriteOptions struct { + Heartbeat time.Duration + Ephemeral bool +} + +// LockOptions contains optional request parameters +type LockOptions struct { + Value []byte // Optional, value to associate with the lock + TTL time.Duration // Optional, expiration ttl associated with the lock +} + +// WatchCallback is used for watch methods on keys +// and is triggered on key change +type WatchCallback func(entries ...*KVPair) + +// Locker provides locking mechanism on top of the store. +// Similar to `sync.Lock` except it may return errors. +type Locker interface { + Lock() (<-chan struct{}, error) + Unlock() error +} diff --git a/Godeps/_workspace/src/github.com/docker/libkv/store/zookeeper/zookeeper.go b/Godeps/_workspace/src/github.com/docker/libkv/store/zookeeper/zookeeper.go new file mode 100644 index 000000000..d12edf679 --- /dev/null +++ b/Godeps/_workspace/src/github.com/docker/libkv/store/zookeeper/zookeeper.go @@ -0,0 +1,379 @@ +package zookeeper + +import ( + "strings" + "time" + + "github.com/docker/libkv/store" + zk "github.com/samuel/go-zookeeper/zk" +) + +const defaultTimeout = 10 * time.Second + +// Zookeeper is the receiver type for +// the Store interface +type Zookeeper struct { + timeout time.Duration + client *zk.Conn +} + +type zookeeperLock struct { + client *zk.Conn + lock *zk.Lock + key string + value []byte +} + +// New creates a new Zookeeper client given a +// list of endpoints and an optional tls config +func New(endpoints []string, options *store.Config) (store.Store, error) { + s := &Zookeeper{} + s.timeout = defaultTimeout + + // Set options + if options != nil { + if options.ConnectionTimeout != 0 { + s.setTimeout(options.ConnectionTimeout) + } + } + + // Connect to Zookeeper + conn, _, err := zk.Connect(endpoints, s.timeout) + if err != nil { + return nil, err + } + s.client = conn + + return s, nil +} + +// setTimeout sets the timeout for connecting to Zookeeper +func (s *Zookeeper) setTimeout(time time.Duration) { + s.timeout = time +} + +// Get the value at "key", returns the last modified index +// to use in conjunction to Atomic calls +func (s *Zookeeper) Get(key string) (pair *store.KVPair, err error) { + resp, meta, err := s.client.Get(store.Normalize(key)) + if err != nil { + return nil, err + } + + // If resp is nil, the key does not exist + if resp == nil { + return nil, store.ErrKeyNotFound + } + + pair = &store.KVPair{ + Key: key, + Value: resp, + LastIndex: uint64(meta.Version), + } + + return pair, nil +} + +// createFullPath creates the entire path for a directory +// that does not exist +func (s *Zookeeper) createFullPath(path []string, ephemeral bool) error { + for i := 1; i <= len(path); i++ { + newpath := "/" + strings.Join(path[:i], "/") + if i == len(path) && ephemeral { + _, err := s.client.Create(newpath, []byte{1}, zk.FlagEphemeral, zk.WorldACL(zk.PermAll)) + return err + } + _, err := s.client.Create(newpath, []byte{1}, 0, zk.WorldACL(zk.PermAll)) + if err != nil { + // Skip if node already exists + if err != zk.ErrNodeExists { + return err + } + } + } + return nil +} + +// Put a value at "key" +func (s *Zookeeper) Put(key string, value []byte, opts *store.WriteOptions) error { + fkey := store.Normalize(key) + + exists, err := s.Exists(key) + if err != nil { + return err + } + + if !exists { + if opts != nil && opts.Ephemeral { + s.createFullPath(store.SplitKey(key), opts.Ephemeral) + } else { + s.createFullPath(store.SplitKey(key), false) + } + } + + _, err = s.client.Set(fkey, value, -1) + return err +} + +// Delete a value at "key" +func (s *Zookeeper) Delete(key string) error { + err := s.client.Delete(store.Normalize(key), -1) + return err +} + +// Exists checks if the key exists inside the store +func (s *Zookeeper) Exists(key string) (bool, error) { + exists, _, err := s.client.Exists(store.Normalize(key)) + if err != nil { + return false, err + } + return exists, nil +} + +// Watch for changes on a "key" +// It returns a channel that will receive changes or pass +// on errors. Upon creation, the current value will first +// be sent to the channel. Providing a non-nil stopCh can +// be used to stop watching. +func (s *Zookeeper) Watch(key string, stopCh <-chan struct{}) (<-chan *store.KVPair, error) { + // Get the key first + pair, err := s.Get(key) + if err != nil { + return nil, err + } + + // Catch zk notifications and fire changes into the channel. + watchCh := make(chan *store.KVPair) + go func() { + defer close(watchCh) + + // Get returns the current value to the channel prior + // to listening to any event that may occur on that key + watchCh <- pair + for { + _, _, eventCh, err := s.client.GetW(store.Normalize(key)) + if err != nil { + return + } + select { + case e := <-eventCh: + if e.Type == zk.EventNodeDataChanged { + if entry, err := s.Get(key); err == nil { + watchCh <- entry + } + } + case <-stopCh: + // There is no way to stop GetW so just quit + return + } + } + }() + + return watchCh, nil +} + +// WatchTree watches for changes on a "directory" +// It returns a channel that will receive changes or pass +// on errors. Upon creating a watch, the current childs values +// will be sent to the channel .Providing a non-nil stopCh can +// be used to stop watching. +func (s *Zookeeper) WatchTree(directory string, stopCh <-chan struct{}) (<-chan []*store.KVPair, error) { + // List the childrens first + entries, err := s.List(directory) + if err != nil { + return nil, err + } + + // Catch zk notifications and fire changes into the channel. + watchCh := make(chan []*store.KVPair) + go func() { + defer close(watchCh) + + // List returns the children values to the channel + // prior to listening to any events that may occur + // on those keys + watchCh <- entries + + for { + _, _, eventCh, err := s.client.ChildrenW(store.Normalize(directory)) + if err != nil { + return + } + select { + case e := <-eventCh: + if e.Type == zk.EventNodeChildrenChanged { + if kv, err := s.List(directory); err == nil { + watchCh <- kv + } + } + case <-stopCh: + // There is no way to stop GetW so just quit + return + } + } + }() + + return watchCh, nil +} + +// List child nodes of a given directory +func (s *Zookeeper) List(directory string) ([]*store.KVPair, error) { + keys, stat, err := s.client.Children(store.Normalize(directory)) + if err != nil { + return nil, err + } + + kv := []*store.KVPair{} + + // FIXME Costly Get request for each child key.. + for _, key := range keys { + pair, err := s.Get(directory + store.Normalize(key)) + if err != nil { + return nil, err + } + + kv = append(kv, &store.KVPair{ + Key: key, + Value: []byte(pair.Value), + LastIndex: uint64(stat.Version), + }) + } + + return kv, nil +} + +// DeleteTree deletes a range of keys under a given directory +func (s *Zookeeper) DeleteTree(directory string) error { + pairs, err := s.List(directory) + if err != nil { + return err + } + + var reqs []interface{} + + for _, pair := range pairs { + reqs = append(reqs, &zk.DeleteRequest{ + Path: store.Normalize(directory + "/" + pair.Key), + Version: -1, + }) + } + + _, err = s.client.Multi(reqs...) + return err +} + +// AtomicPut put a value at "key" if the key has not been +// modified in the meantime, throws an error if this is the case +func (s *Zookeeper) AtomicPut(key string, value []byte, previous *store.KVPair, _ *store.WriteOptions) (bool, *store.KVPair, error) { + + var lastIndex uint64 + if previous != nil { + meta, err := s.client.Set(store.Normalize(key), value, int32(previous.LastIndex)) + if err != nil { + // Compare Failed + if err == zk.ErrBadVersion { + return false, nil, store.ErrKeyModified + } + return false, nil, err + } + lastIndex = uint64(meta.Version) + } else { + // Interpret previous == nil as create operation. + _, err := s.client.Create(store.Normalize(key), value, 0, zk.WorldACL(zk.PermAll)) + if err != nil { + // Zookeeper will complain if the directory doesn't exist. + if err == zk.ErrNoNode { + // Create the directory + parts := store.SplitKey(key) + parts = parts[:len(parts)-1] + if err = s.createFullPath(parts, false); err != nil { + // Failed to create the directory. + return false, nil, err + } + if _, err := s.client.Create(store.Normalize(key), value, 0, zk.WorldACL(zk.PermAll)); err != nil { + return false, nil, err + } + + } else { + // Unhandled error + return false, nil, err + } + } + lastIndex = 0 // Newly created nodes have version 0. + } + + pair := &store.KVPair{ + Key: key, + Value: value, + LastIndex: lastIndex, + } + + return true, pair, nil +} + +// AtomicDelete deletes a value at "key" if the key +// has not been modified in the meantime, throws an +// error if this is the case +func (s *Zookeeper) AtomicDelete(key string, previous *store.KVPair) (bool, error) { + if previous == nil { + return false, store.ErrPreviousNotSpecified + } + + err := s.client.Delete(store.Normalize(key), int32(previous.LastIndex)) + if err != nil { + if err == zk.ErrBadVersion { + return false, store.ErrKeyModified + } + return false, err + } + return true, nil +} + +// NewLock returns a handle to a lock struct which can +// be used to provide mutual exclusion on a key +func (s *Zookeeper) NewLock(key string, options *store.LockOptions) (lock store.Locker, err error) { + value := []byte("") + + // Apply options + if options != nil { + if options.Value != nil { + value = options.Value + } + } + + lock = &zookeeperLock{ + client: s.client, + key: store.Normalize(key), + value: value, + lock: zk.NewLock(s.client, store.Normalize(key), zk.WorldACL(zk.PermAll)), + } + + return lock, err +} + +// Lock attempts to acquire the lock and blocks while +// doing so. It returns a channel that is closed if our +// lock is lost or if an error occurs +func (l *zookeeperLock) Lock() (<-chan struct{}, error) { + err := l.lock.Lock() + + if err == nil { + // We hold the lock, we can set our value + // FIXME: The value is left behind + // (problematic for leader election) + _, err = l.client.Set(l.key, l.value, -1) + } + + return make(chan struct{}), err +} + +// Unlock the "key". Calling unlock while +// not holding the lock will throw an error +func (l *zookeeperLock) Unlock() error { + return l.lock.Unlock() +} + +// Close closes the client connection +func (s *Zookeeper) Close() { + s.client.Close() +} diff --git a/Godeps/_workspace/src/github.com/docker/libkv/store/zookeeper/zookeeper_test.go b/Godeps/_workspace/src/github.com/docker/libkv/store/zookeeper/zookeeper_test.go new file mode 100644 index 000000000..759297e54 --- /dev/null +++ b/Godeps/_workspace/src/github.com/docker/libkv/store/zookeeper/zookeeper_test.go @@ -0,0 +1,34 @@ +package zookeeper + +import ( + "testing" + "time" + + "github.com/docker/libkv/store" + "github.com/docker/libkv/testutils" +) + +func makeZkClient(t *testing.T) store.Store { + client := "localhost:2181" + + kv, err := New( + []string{client}, + &store.Config{ + ConnectionTimeout: 3 * time.Second, + EphemeralTTL: 2 * time.Second, + }, + ) + + if err != nil { + t.Fatalf("cannot create store: %v", err) + } + + return kv +} + +func TestZkStore(t *testing.T) { + kv := makeZkClient(t) + backup := makeZkClient(t) + + testutils.RunTestStore(t, kv, backup) +} diff --git a/Godeps/_workspace/src/github.com/docker/libkv/testutils/utils.go b/Godeps/_workspace/src/github.com/docker/libkv/testutils/utils.go new file mode 100644 index 000000000..717d9ecdc --- /dev/null +++ b/Godeps/_workspace/src/github.com/docker/libkv/testutils/utils.go @@ -0,0 +1,430 @@ +package testutils + +import ( + "testing" + "time" + + "github.com/docker/libkv/store" + "github.com/stretchr/testify/assert" +) + +// RunTestStore is an helper testing method that is +// called by each K/V backend sub-package testing +func RunTestStore(t *testing.T, kv store.Store, backup store.Store) { + testPutGetDelete(t, kv) + testWatch(t, kv) + testWatchTree(t, kv) + testAtomicPut(t, kv) + testAtomicPutCreate(t, kv) + testAtomicDelete(t, kv) + testLockUnlock(t, kv) + testPutEphemeral(t, kv, backup) + testList(t, kv) + testDeleteTree(t, kv) +} + +func testPutGetDelete(t *testing.T, kv store.Store) { + key := "foo" + value := []byte("bar") + + // Put the key + err := kv.Put(key, value, nil) + assert.NoError(t, err) + + // Get should return the value and an incremented index + pair, err := kv.Get(key) + assert.NoError(t, err) + if assert.NotNil(t, pair) { + assert.NotNil(t, pair.Value) + } + assert.Equal(t, pair.Value, value) + assert.NotEqual(t, pair.LastIndex, 0) + + // Delete the key + err = kv.Delete(key) + assert.NoError(t, err) + + // Get should fail + pair, err = kv.Get(key) + assert.Error(t, err) + assert.Nil(t, pair) +} + +func testWatch(t *testing.T, kv store.Store) { + key := "hello" + value := []byte("world") + newValue := []byte("world!") + + // Put the key + err := kv.Put(key, value, nil) + assert.NoError(t, err) + + stopCh := make(<-chan struct{}) + events, err := kv.Watch(key, stopCh) + assert.NoError(t, err) + assert.NotNil(t, events) + + // Update loop + go func() { + timeout := time.After(1 * time.Second) + tick := time.Tick(250 * time.Millisecond) + for { + select { + case <-timeout: + return + case <-tick: + err := kv.Put(key, newValue, nil) + if assert.NoError(t, err) { + continue + } + return + } + } + }() + + // Check for updates + eventCount := 1 + for { + select { + case event := <-events: + assert.NotNil(t, event) + if eventCount == 1 { + assert.Equal(t, event.Key, key) + assert.Equal(t, event.Value, value) + } else { + assert.Equal(t, event.Key, key) + assert.Equal(t, event.Value, newValue) + } + eventCount++ + // We received all the events we wanted to check + if eventCount >= 4 { + return + } + case <-time.After(4 * time.Second): + t.Fatal("Timeout reached") + return + } + } +} + +func testWatchTree(t *testing.T, kv store.Store) { + dir := "tree" + + node1 := "tree/node1" + value1 := []byte("node1") + + node2 := "tree/node2" + value2 := []byte("node2") + + node3 := "tree/node3" + value3 := []byte("node3") + + err := kv.Put(node1, value1, nil) + assert.NoError(t, err) + err = kv.Put(node2, value2, nil) + assert.NoError(t, err) + err = kv.Put(node3, value3, nil) + assert.NoError(t, err) + + stopCh := make(<-chan struct{}) + events, err := kv.WatchTree(dir, stopCh) + assert.NoError(t, err) + assert.NotNil(t, events) + + // Update loop + go func() { + timeout := time.After(250 * time.Millisecond) + for { + select { + case <-timeout: + err := kv.Delete(node3) + assert.NoError(t, err) + return + } + } + }() + + // Check for updates + for { + select { + case event := <-events: + assert.NotNil(t, event) + // We received the Delete event on a child node + // Exit test successfully + if len(event) == 2 { + return + } + case <-time.After(4 * time.Second): + t.Fatal("Timeout reached") + return + } + } +} + +func testAtomicPut(t *testing.T, kv store.Store) { + key := "hello" + value := []byte("world") + + // Put the key + err := kv.Put(key, value, nil) + assert.NoError(t, err) + + // Get should return the value and an incremented index + pair, err := kv.Get(key) + assert.NoError(t, err) + if assert.NotNil(t, pair) { + assert.NotNil(t, pair.Value) + } + assert.Equal(t, pair.Value, value) + assert.NotEqual(t, pair.LastIndex, 0) + + // This CAS should fail: previous exists. + success, _, err := kv.AtomicPut("hello", []byte("WORLD"), nil, nil) + assert.Error(t, err) + assert.False(t, success) + + // This CAS should succeed + success, _, err = kv.AtomicPut("hello", []byte("WORLD"), pair, nil) + assert.NoError(t, err) + assert.True(t, success) + + // This CAS should fail, key exists. + pair.LastIndex = 0 + success, _, err = kv.AtomicPut("hello", []byte("WORLDWORLD"), pair, nil) + assert.Error(t, err) + assert.False(t, success) +} + +func testAtomicPutCreate(t *testing.T, kv store.Store) { + // Use a key in a new directory to ensure Stores will create directories + // that don't yet exist. + key := "put/create" + value := []byte("putcreate") + + // AtomicPut the key, previous = nil indicates create. + success, _, err := kv.AtomicPut(key, value, nil, nil) + assert.NoError(t, err) + assert.True(t, success) + + // Get should return the value and an incremented index + pair, err := kv.Get(key) + assert.NoError(t, err) + if assert.NotNil(t, pair) { + assert.NotNil(t, pair.Value) + } + assert.Equal(t, pair.Value, value) + + // Attempting to create again should fail. + success, _, err = kv.AtomicPut(key, value, nil, nil) + assert.Error(t, err) + assert.False(t, success) + + // This CAS should succeed, since it has the value from Get() + success, _, err = kv.AtomicPut(key, []byte("PUTCREATE"), pair, nil) + assert.NoError(t, err) + assert.True(t, success) + + // Delete the key, ensures runs of the test don't interfere with each other. + err = kv.DeleteTree("put") + assert.NoError(t, err) +} + +func testAtomicDelete(t *testing.T, kv store.Store) { + key := "atomic" + value := []byte("world") + + // Put the key + err := kv.Put(key, value, nil) + assert.NoError(t, err) + + // Get should return the value and an incremented index + pair, err := kv.Get(key) + assert.NoError(t, err) + if assert.NotNil(t, pair) { + assert.NotNil(t, pair.Value) + } + assert.Equal(t, pair.Value, value) + assert.NotEqual(t, pair.LastIndex, 0) + + tempIndex := pair.LastIndex + + // AtomicDelete should fail + pair.LastIndex = 0 + success, err := kv.AtomicDelete(key, pair) + assert.Error(t, err) + assert.False(t, success) + + // AtomicDelete should succeed + pair.LastIndex = tempIndex + success, err = kv.AtomicDelete(key, pair) + assert.NoError(t, err) + assert.True(t, success) +} + +func testLockUnlock(t *testing.T, kv store.Store) { + key := "foo" + value := []byte("bar") + + // We should be able to create a new lock on key + lock, err := kv.NewLock(key, &store.LockOptions{Value: value}) + assert.NoError(t, err) + assert.NotNil(t, lock) + + // Lock should successfully succeed or block + lockChan, err := lock.Lock() + assert.NoError(t, err) + assert.NotNil(t, lockChan) + + // Get should work + pair, err := kv.Get(key) + assert.NoError(t, err) + if assert.NotNil(t, pair) { + assert.NotNil(t, pair.Value) + } + assert.Equal(t, pair.Value, value) + assert.NotEqual(t, pair.LastIndex, 0) + + // Unlock should succeed + err = lock.Unlock() + assert.NoError(t, err) + + // Get should work + pair, err = kv.Get(key) + assert.NoError(t, err) + if assert.NotNil(t, pair) { + assert.NotNil(t, pair.Value) + } + assert.Equal(t, pair.Value, value) + assert.NotEqual(t, pair.LastIndex, 0) +} + +func testPutEphemeral(t *testing.T, kv store.Store, otherConn store.Store) { + firstKey := "first" + firstValue := []byte("foo") + + secondKey := "second" + secondValue := []byte("bar") + + // Put the first key with the Ephemeral flag + err := otherConn.Put(firstKey, firstValue, &store.WriteOptions{Ephemeral: true}) + assert.NoError(t, err) + + // Put a second key with the Ephemeral flag + err = otherConn.Put(secondKey, secondValue, &store.WriteOptions{Ephemeral: true}) + assert.NoError(t, err) + + // Get on firstKey should work + pair, err := kv.Get(firstKey) + assert.NoError(t, err) + assert.NotNil(t, pair) + + // Get on secondKey should work + pair, err = kv.Get(secondKey) + assert.NoError(t, err) + assert.NotNil(t, pair) + + // Close the connection + otherConn.Close() + + // Let the session expire + time.Sleep(3 * time.Second) + + // Get on firstKey shouldn't work + pair, err = kv.Get(firstKey) + assert.Error(t, err) + assert.Nil(t, pair) + + // Get on secondKey shouldn't work + pair, err = kv.Get(secondKey) + assert.Error(t, err) + assert.Nil(t, pair) +} + +func testList(t *testing.T, kv store.Store) { + prefix := "nodes" + + firstKey := "nodes/first" + firstValue := []byte("first") + + secondKey := "nodes/second" + secondValue := []byte("second") + + // Put the first key + err := kv.Put(firstKey, firstValue, nil) + assert.NoError(t, err) + + // Put the second key + err = kv.Put(secondKey, secondValue, nil) + assert.NoError(t, err) + + // List should work and return the two correct values + pairs, err := kv.List(prefix) + assert.NoError(t, err) + if assert.NotNil(t, pairs) { + assert.Equal(t, len(pairs), 2) + } + + // Check pairs, those are not necessarily in Put order + for _, pair := range pairs { + if pair.Key == firstKey { + assert.Equal(t, pair.Value, firstValue) + } + if pair.Key == secondKey { + assert.Equal(t, pair.Value, secondValue) + } + } + + // List should fail: the key does not exist + pairs, err = kv.List("idontexist") + assert.Error(t, err) + assert.Nil(t, pairs) +} + +func testDeleteTree(t *testing.T, kv store.Store) { + prefix := "nodes" + + firstKey := "nodes/first" + firstValue := []byte("first") + + secondKey := "nodes/second" + secondValue := []byte("second") + + // Put the first key + err := kv.Put(firstKey, firstValue, nil) + assert.NoError(t, err) + + // Put the second key + err = kv.Put(secondKey, secondValue, nil) + assert.NoError(t, err) + + // Get should work on the first Key + pair, err := kv.Get(firstKey) + assert.NoError(t, err) + if assert.NotNil(t, pair) { + assert.NotNil(t, pair.Value) + } + assert.Equal(t, pair.Value, firstValue) + assert.NotEqual(t, pair.LastIndex, 0) + + // Get should work on the second Key + pair, err = kv.Get(secondKey) + assert.NoError(t, err) + if assert.NotNil(t, pair) { + assert.NotNil(t, pair.Value) + } + assert.Equal(t, pair.Value, secondValue) + assert.NotEqual(t, pair.LastIndex, 0) + + // Delete Values under directory `nodes` + err = kv.DeleteTree(prefix) + assert.NoError(t, err) + + // Get should fail on both keys + pair, err = kv.Get(firstKey) + assert.Error(t, err) + assert.Nil(t, pair) + + pair, err = kv.Get(secondKey) + assert.Error(t, err) + assert.Nil(t, pair) +} diff --git a/Godeps/_workspace/src/github.com/docker/libnetwork/.gitignore b/Godeps/_workspace/src/github.com/docker/libnetwork/.gitignore new file mode 100644 index 000000000..c03c9653a --- /dev/null +++ b/Godeps/_workspace/src/github.com/docker/libnetwork/.gitignore @@ -0,0 +1,33 @@ +# Compiled Object files, Static and Dynamic libs (Shared Objects) +*.o +*.a +*.so + +# Folders +_obj +_test + +# Architecture specific extensions/prefixes +*.[568vq] +[568vq].out + +*.cgo1.go +*.cgo2.c +_cgo_defun.c +_cgo_gotypes.go +_cgo_export.* + +_testmain.go + +*.exe +*.test +*.prof + +# Coverage +*.tmp +*.coverprofile + +# IDE files +.project + +libnetwork-build.created diff --git a/Godeps/_workspace/src/github.com/docker/libnetwork/Godeps/Godeps.json b/Godeps/_workspace/src/github.com/docker/libnetwork/Godeps/Godeps.json new file mode 100644 index 000000000..23b929562 --- /dev/null +++ b/Godeps/_workspace/src/github.com/docker/libnetwork/Godeps/Godeps.json @@ -0,0 +1,137 @@ +{ + "ImportPath": "github.com/docker/libnetwork", + "GoVersion": "go1.4.2", + "Packages": [ + "./..." + ], + "Deps": [ + { + "ImportPath": "github.com/BurntSushi/toml", + "Comment": "v0.1.0-16-gf706d00", + "Rev": "f706d00e3de6abe700c994cdd545a1a4915af060" + }, + { + "ImportPath": "github.com/Sirupsen/logrus", + "Comment": "v0.6.4-12-g467d9d5", + "Rev": "467d9d55c2d2c17248441a8fc661561161f40d5e" + }, + { + "ImportPath": "github.com/armon/go-metrics", + "Rev": "eb0af217e5e9747e41dd5303755356b62d28e3ec" + }, + { + "ImportPath": "github.com/coreos/go-etcd/etcd", + "Comment": "v2.0.0-7-g73a8ef7", + "Rev": "73a8ef737e8ea002281a28b4cb92a1de121ad4c6" + }, + { + "ImportPath": "github.com/docker/docker/pkg/homedir", + "Comment": "v1.4.1-4106-g637023a", + "Rev": "637023a5f8d8347a0e271c09d5c9bc84fbc97693" + }, + { + "ImportPath": "github.com/docker/docker/pkg/ioutils", + "Comment": "v1.4.1-4106-g637023a", + "Rev": "637023a5f8d8347a0e271c09d5c9bc84fbc97693" + }, + { + "ImportPath": "github.com/docker/docker/pkg/mflag", + "Comment": "v1.4.1-4106-g637023a", + "Rev": "637023a5f8d8347a0e271c09d5c9bc84fbc97693" + }, + { + "ImportPath": "github.com/docker/docker/pkg/parsers", + "Comment": "v1.4.1-4106-g637023a", + "Rev": "637023a5f8d8347a0e271c09d5c9bc84fbc97693" + }, + { + "ImportPath": "github.com/docker/docker/pkg/plugins", + "Comment": "v1.4.1-4106-g637023a", + "Rev": "637023a5f8d8347a0e271c09d5c9bc84fbc97693" + }, + { + "ImportPath": "github.com/docker/docker/pkg/proxy", + "Comment": "v1.4.1-4106-g637023a", + "Rev": "637023a5f8d8347a0e271c09d5c9bc84fbc97693" + }, + { + "ImportPath": "github.com/docker/docker/pkg/reexec", + "Comment": "v1.4.1-4106-g637023a", + "Rev": "637023a5f8d8347a0e271c09d5c9bc84fbc97693" + }, + { + "ImportPath": "github.com/docker/docker/pkg/stringid", + "Comment": "v1.4.1-4106-g637023a", + "Rev": "637023a5f8d8347a0e271c09d5c9bc84fbc97693" + }, + { + "ImportPath": "github.com/docker/docker/pkg/term", + "Comment": "v1.4.1-4106-g637023a", + "Rev": "637023a5f8d8347a0e271c09d5c9bc84fbc97693" + }, + { + "ImportPath": "github.com/docker/libcontainer/user", + "Comment": "v1.4.0-495-g3e66118", + "Rev": "3e661186ba24f259d3860f067df052c7f6904bee" + }, + { + "ImportPath": "github.com/docker/libkv", + "Rev": "60c7c881345b3c67defc7f93a8297debf041d43c" + }, + { + "ImportPath": "github.com/godbus/dbus", + "Comment": "v2-3-g4160802", + "Rev": "41608027bdce7bfa8959d653a00b954591220e67" + }, + { + "ImportPath": "github.com/gorilla/context", + "Rev": "215affda49addc4c8ef7e2534915df2c8c35c6cd" + }, + { + "ImportPath": "github.com/gorilla/mux", + "Rev": "8096f47503459bcc74d1f4c487b7e6e42e5746b5" + }, + { + "ImportPath": "github.com/hashicorp/consul/api", + "Comment": "v0.5.0rc1-66-g954aec6", + "Rev": "954aec66231b79c161a4122b023fbcad13047f79" + }, + { + "ImportPath": "github.com/hashicorp/go-msgpack/codec", + "Rev": "71c2886f5a673a35f909803f38ece5810165097b" + }, + { + "ImportPath": "github.com/hashicorp/memberlist", + "Rev": "9a1e242e454d2443df330bdd51a436d5a9058fc4" + }, + { + "ImportPath": "github.com/hashicorp/serf/serf", + "Comment": "v0.6.4", + "Rev": "7151adcef72687bf95f451a2e0ba15cb19412bf2" + }, + { + "ImportPath": "github.com/samuel/go-zookeeper/zk", + "Rev": "d0e0d8e11f318e000a8cc434616d69e329edc374" + }, + { + "ImportPath": "github.com/stretchr/objx", + "Rev": "cbeaeb16a013161a98496fad62933b1d21786672" + }, + { + "ImportPath": "github.com/stretchr/testify/assert", + "Rev": "dab07ac62d4905d3e48d17dc549c684ac3b7c15a" + }, + { + "ImportPath": "github.com/stretchr/testify/mock", + "Rev": "dab07ac62d4905d3e48d17dc549c684ac3b7c15a" + }, + { + "ImportPath": "github.com/vishvananda/netns", + "Rev": "493029407eeb434d0c2d44e02ea072ff2488d322" + }, + { + "ImportPath": "github.com/vishvananda/netlink", + "Rev": "4b5dce31de6d42af5bb9811c6d265472199e0fec" + } + ] +} diff --git a/Godeps/_workspace/src/github.com/docker/libnetwork/Godeps/Readme b/Godeps/_workspace/src/github.com/docker/libnetwork/Godeps/Readme new file mode 100644 index 000000000..4cdaa53d5 --- /dev/null +++ b/Godeps/_workspace/src/github.com/docker/libnetwork/Godeps/Readme @@ -0,0 +1,5 @@ +This directory tree is generated automatically by godep. + +Please do not edit. + +See https://github.com/tools/godep for more information. diff --git a/Godeps/_workspace/src/github.com/docker/libnetwork/LICENSE b/Godeps/_workspace/src/github.com/docker/libnetwork/LICENSE new file mode 100644 index 000000000..e06d20818 --- /dev/null +++ b/Godeps/_workspace/src/github.com/docker/libnetwork/LICENSE @@ -0,0 +1,202 @@ +Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + diff --git a/Godeps/_workspace/src/github.com/docker/libnetwork/MAINTAINERS b/Godeps/_workspace/src/github.com/docker/libnetwork/MAINTAINERS new file mode 100644 index 000000000..69f1e9b88 --- /dev/null +++ b/Godeps/_workspace/src/github.com/docker/libnetwork/MAINTAINERS @@ -0,0 +1,5 @@ +Alessandro Boch (@aboch) +Alexandr Morozov (@LK4D4) +Arnaud Porterie (@icecrime) +Jana Radhakrishnan (@mrjana) +Madhu Venugopal (@mavenugo) diff --git a/Godeps/_workspace/src/github.com/docker/libnetwork/Makefile b/Godeps/_workspace/src/github.com/docker/libnetwork/Makefile new file mode 100644 index 000000000..deb510cf8 --- /dev/null +++ b/Godeps/_workspace/src/github.com/docker/libnetwork/Makefile @@ -0,0 +1,79 @@ +.PHONY: all all-local build build-local check check-code check-format run-tests check-local install-deps coveralls circle-ci +SHELL=/bin/bash +build_image=libnetwork-build +dockerargs = --privileged -v $(shell pwd):/go/src/github.com/docker/libnetwork -w /go/src/github.com/docker/libnetwork +container_env = -e "INSIDECONTAINER=-incontainer=true" +docker = docker run --rm ${dockerargs} ${container_env} ${build_image} +ciargs = -e "COVERALLS_TOKEN=$$COVERALLS_TOKEN" -e "INSIDECONTAINER=-incontainer=true" +cidocker = docker run ${ciargs} ${dockerargs} golang:1.4 + +all: ${build_image}.created + ${docker} make all-local + +all-local: check-local build-local + +${build_image}.created: + docker run --name=libnetworkbuild -v $(shell pwd):/go/src/github.com/docker/libnetwork -w /go/src/github.com/docker/libnetwork golang:1.4 make install-deps + docker commit libnetworkbuild ${build_image} + docker rm libnetworkbuild + touch ${build_image}.created + +build: ${build_image}.created + ${docker} make build-local + +build-local: + $(shell which godep) go build -tags libnetwork_discovery ./... + +check: ${build_image}.created + ${docker} make check-local + +check-code: + @echo "Checking code... " + test -z "$$(golint ./... | tee /dev/stderr)" + go vet ./... + @echo "Done checking code" + +check-format: + @echo "Checking format... " + test -z "$$(goimports -l . | grep -v Godeps/_workspace/src/ | tee /dev/stderr)" + @echo "Done checking format" + +run-tests: + @echo "Running tests... " + @echo "mode: count" > coverage.coverprofile + @for dir in $$(find . -maxdepth 10 -not -path './.git*' -not -path '*/_*' -type d); do \ + if ls $$dir/*.go &> /dev/null; then \ + pushd . &> /dev/null ; \ + cd $$dir ; \ + $(shell which godep) go test ${INSIDECONTAINER} -test.parallel 3 -test.v -covermode=count -coverprofile=./profile.tmp ; \ + ret=$$? ;\ + if [ $$ret -ne 0 ]; then exit $$ret; fi ;\ + popd &> /dev/null; \ + if [ -f $$dir/profile.tmp ]; then \ + cat $$dir/profile.tmp | tail -n +2 >> coverage.coverprofile ; \ + rm $$dir/profile.tmp ; \ + fi ; \ + fi ; \ + done + @echo "Done running tests" + +check-local: check-format check-code run-tests + +install-deps: + apt-get update && apt-get -y install iptables + git clone https://github.com/golang/tools /go/src/golang.org/x/tools + go install golang.org/x/tools/cmd/vet + go install golang.org/x/tools/cmd/goimports + go install golang.org/x/tools/cmd/cover + go get github.com/tools/godep + go get github.com/golang/lint/golint + go get github.com/mattn/goveralls + +coveralls: + -@goveralls -service circleci -coverprofile=coverage.coverprofile -repotoken $$COVERALLS_TOKEN + +# CircleCI's Docker fails when cleaning up using the --rm flag +# The following target is a workaround for this + +circle-ci: + @${cidocker} make install-deps check-local coveralls diff --git a/Godeps/_workspace/src/github.com/docker/libnetwork/README.md b/Godeps/_workspace/src/github.com/docker/libnetwork/README.md new file mode 100644 index 000000000..d71c5b31e --- /dev/null +++ b/Godeps/_workspace/src/github.com/docker/libnetwork/README.md @@ -0,0 +1,88 @@ +# libnetwork - networking for containers + +[![Circle CI](https://circleci.com/gh/docker/libnetwork/tree/master.svg?style=svg)](https://circleci.com/gh/docker/libnetwork/tree/master) [![Coverage Status](https://coveralls.io/repos/docker/libnetwork/badge.svg)](https://coveralls.io/r/docker/libnetwork) [![GoDoc](https://godoc.org/github.com/docker/libnetwork?status.svg)](https://godoc.org/github.com/docker/libnetwork) + +Libnetwork provides a native Go implementation for connecting containers + +The goal of libnetwork is to deliver a robust Container Network Model that provides a consistent programming interface and the required network abstractions for applications. + +**NOTE**: libnetwork project is under heavy development and is not ready for general use. + +#### Design +Please refer to the [design](docs/design.md) for more information. + +#### Using libnetwork + +There are many networking solutions available to suit a broad range of use-cases. libnetwork uses a driver / plugin model to support all of these solutions while abstracting the complexity of the driver implementations by exposing a simple and consistent Network Model to users. + + +```go + // Create a new controller instance + controller, err := libnetwork.New() + if err != nil { + return + } + + // Select and configure the network driver + networkType := "bridge" + + driverOptions := options.Generic{} + genericOption := make(map[string]interface{}) + genericOption[netlabel.GenericData] = driverOptions + err := controller.ConfigureNetworkDriver(networkType, genericOption) + if err != nil { + return + } + + // Create a network for containers to join. + // NewNetwork accepts Variadic optional arguments that libnetwork and Drivers can make of + network, err := controller.NewNetwork(networkType, "network1") + if err != nil { + return + } + + // For each new container: allocate IP and interfaces. The returned network + // settings will be used for container infos (inspect and such), as well as + // iptables rules for port publishing. This info is contained or accessible + // from the returned endpoint. + ep, err := network.CreateEndpoint("Endpoint1") + if err != nil { + return + } + + // A container can join the endpoint by providing the container ID to the join + // api. + // Join accepts Variadic arguments which will be made use of by libnetwork and Drivers + err = ep.Join("container1", + libnetwork.JoinOptionHostname("test"), + libnetwork.JoinOptionDomainname("docker.io")) + if err != nil { + return + } + + // libnetwork client can check the endpoint's operational data via the Info() API + epInfo, err := ep.DriverInfo() + mapData, ok := epInfo[netlabel.PortMap] + if ok { + portMapping, ok := mapData.([]types.PortBinding) + if ok { + fmt.Printf("Current port mapping for endpoint %s: %v", ep.Name(), portMapping) + } + } + +``` +#### Current Status +Please watch this space for updates on the progress. + +Currently libnetwork is nothing more than an attempt to modularize the Docker platform's networking subsystem by moving it into libnetwork as a library. + +## Future +Please refer to [roadmap](ROADMAP.md) for more information. + +## Contributing + +Want to hack on libnetwork? [Docker's contributions guidelines](https://github.com/docker/docker/blob/master/CONTRIBUTING.md) apply. + +## Copyright and license +Code and documentation copyright 2015 Docker, inc. Code released under the Apache 2.0 license. Docs released under Creative commons. + diff --git a/Godeps/_workspace/src/github.com/docker/libnetwork/ROADMAP.md b/Godeps/_workspace/src/github.com/docker/libnetwork/ROADMAP.md new file mode 100644 index 000000000..9cb3174ec --- /dev/null +++ b/Godeps/_workspace/src/github.com/docker/libnetwork/ROADMAP.md @@ -0,0 +1,20 @@ +# Roadmap + +This document defines the high-level goals of the libnetwork project. See [Project Planning](#project-planning) for information on Releases. + +## Long-term Goal + +libnetwork project will follow Docker and Linux philosophy of delivering small, highly modular and composable tools that works well independently. +libnetwork aims to satisfy that composable need for Networking in Containers. + +## Short-term Goals + +- Modularize the networking logic in Docker Engine and libcontainer in to a single, reusable library +- Replace the networking subsystem of Docker Engine, with libnetwork +- Define a flexible model that allows local and remote drivers to provide networking to containers +- Provide a stand-alone tool "dnet" for managing and testing libnetwork + +Project Planning +================ + +[Project Pages](https://github.com/docker/libnetwork/wiki) define the goals for each Milestone and identify the release-relationship to the Docker Platform. diff --git a/Godeps/_workspace/src/github.com/docker/libnetwork/api/api.go b/Godeps/_workspace/src/github.com/docker/libnetwork/api/api.go new file mode 100644 index 000000000..2bfc81b07 --- /dev/null +++ b/Godeps/_workspace/src/github.com/docker/libnetwork/api/api.go @@ -0,0 +1,796 @@ +package api + +import ( + "encoding/json" + "fmt" + "io/ioutil" + "net/http" + "strings" + + "github.com/docker/libnetwork" + "github.com/docker/libnetwork/netlabel" + "github.com/docker/libnetwork/types" + "github.com/gorilla/mux" +) + +var ( + successResponse = responseStatus{Status: "Success", StatusCode: http.StatusOK} + createdResponse = responseStatus{Status: "Created", StatusCode: http.StatusCreated} + mismatchResponse = responseStatus{Status: "Body/URI parameter mismatch", StatusCode: http.StatusBadRequest} + badQueryResponse = responseStatus{Status: "Unsupported query", StatusCode: http.StatusBadRequest} +) + +const ( + // Resource name regex + // Gorilla mux encloses the passed pattern with '^' and '$'. So we need to do some tricks + // to have mux eventually build a query regex which matches empty or word string (`^$|[\w]+`) + regex = "[a-zA-Z_0-9-]+" + qregx = "$|" + regex + // Router URL variable definition + nwName = "{" + urlNwName + ":" + regex + "}" + nwNameQr = "{" + urlNwName + ":" + qregx + "}" + nwID = "{" + urlNwID + ":" + regex + "}" + nwPIDQr = "{" + urlNwPID + ":" + qregx + "}" + epName = "{" + urlEpName + ":" + regex + "}" + epNameQr = "{" + urlEpName + ":" + qregx + "}" + epID = "{" + urlEpID + ":" + regex + "}" + epPIDQr = "{" + urlEpPID + ":" + qregx + "}" + cnID = "{" + urlCnID + ":" + regex + "}" + + // Internal URL variable name.They can be anything as + // long as they do not collide with query fields. + urlNwName = "network-name" + urlNwID = "network-id" + urlNwPID = "network-partial-id" + urlEpName = "endpoint-name" + urlEpID = "endpoint-id" + urlEpPID = "endpoint-partial-id" + urlCnID = "container-id" + + // BridgeNetworkDriver is the built-in default for Network Driver + BridgeNetworkDriver = "bridge" +) + +// NewHTTPHandler creates and initialize the HTTP handler to serve the requests for libnetwork +func NewHTTPHandler(c libnetwork.NetworkController) func(w http.ResponseWriter, req *http.Request) { + h := &httpHandler{c: c} + h.initRouter() + return h.handleRequest +} + +type responseStatus struct { + Status string + StatusCode int +} + +func (r *responseStatus) isOK() bool { + return r.StatusCode == http.StatusOK || r.StatusCode == http.StatusCreated +} + +type processor func(c libnetwork.NetworkController, vars map[string]string, body []byte) (interface{}, *responseStatus) + +type httpHandler struct { + c libnetwork.NetworkController + r *mux.Router +} + +func (h *httpHandler) handleRequest(w http.ResponseWriter, req *http.Request) { + // Make sure the service is there + if h.c == nil { + http.Error(w, "NetworkController is not available", http.StatusServiceUnavailable) + return + } + + // Get handler from router and execute it + h.r.ServeHTTP(w, req) +} + +func (h *httpHandler) initRouter() { + m := map[string][]struct { + url string + qrs []string + fct processor + }{ + "GET": { + // Order matters + {"/networks", []string{"name", nwNameQr}, procGetNetworks}, + {"/networks", []string{"partial-id", nwPIDQr}, procGetNetworks}, + {"/networks", nil, procGetNetworks}, + {"/networks/" + nwID, nil, procGetNetwork}, + {"/networks/" + nwID + "/endpoints", []string{"name", epNameQr}, procGetEndpoints}, + {"/networks/" + nwID + "/endpoints", []string{"partial-id", epPIDQr}, procGetEndpoints}, + {"/networks/" + nwID + "/endpoints", nil, procGetEndpoints}, + {"/networks/" + nwID + "/endpoints/" + epID, nil, procGetEndpoint}, + {"/services", []string{"network", nwNameQr}, procGetServices}, + {"/services", []string{"name", epNameQr}, procGetServices}, + {"/services", []string{"partial-id", epPIDQr}, procGetServices}, + {"/services", nil, procGetServices}, + {"/services/" + epID, nil, procGetService}, + {"/services/" + epID + "/backend", nil, procGetContainers}, + }, + "POST": { + {"/networks", nil, procCreateNetwork}, + {"/networks/" + nwID + "/endpoints", nil, procCreateEndpoint}, + {"/networks/" + nwID + "/endpoints/" + epID + "/containers", nil, procJoinEndpoint}, + {"/services", nil, procPublishService}, + {"/services/" + epID + "/backend", nil, procAttachBackend}, + }, + "DELETE": { + {"/networks/" + nwID, nil, procDeleteNetwork}, + {"/networks/" + nwID + "/endpoints/" + epID, nil, procDeleteEndpoint}, + {"/networks/" + nwID + "/endpoints/" + epID + "/containers/" + cnID, nil, procLeaveEndpoint}, + {"/services/" + epID, nil, procUnpublishService}, + {"/services/" + epID + "/backend/" + cnID, nil, procDetachBackend}, + }, + } + + h.r = mux.NewRouter() + for method, routes := range m { + for _, route := range routes { + r := h.r.Path("/{.*}" + route.url).Methods(method).HandlerFunc(makeHandler(h.c, route.fct)) + if route.qrs != nil { + r.Queries(route.qrs...) + } + + r = h.r.Path(route.url).Methods(method).HandlerFunc(makeHandler(h.c, route.fct)) + if route.qrs != nil { + r.Queries(route.qrs...) + } + } + } +} + +func makeHandler(ctrl libnetwork.NetworkController, fct processor) http.HandlerFunc { + return func(w http.ResponseWriter, req *http.Request) { + var ( + body []byte + err error + ) + if req.Body != nil { + body, err = ioutil.ReadAll(req.Body) + if err != nil { + http.Error(w, "Invalid body: "+err.Error(), http.StatusBadRequest) + return + } + } + + res, rsp := fct(ctrl, mux.Vars(req), body) + if res != nil { + writeJSON(w, rsp.StatusCode, res) + } + } +} + +/***************** + Resource Builders +******************/ + +func buildNetworkResource(nw libnetwork.Network) *networkResource { + r := &networkResource{} + if nw != nil { + r.Name = nw.Name() + r.ID = nw.ID() + r.Type = nw.Type() + epl := nw.Endpoints() + r.Endpoints = make([]*endpointResource, 0, len(epl)) + for _, e := range epl { + epr := buildEndpointResource(e) + r.Endpoints = append(r.Endpoints, epr) + } + } + return r +} + +func buildEndpointResource(ep libnetwork.Endpoint) *endpointResource { + r := &endpointResource{} + if ep != nil { + r.Name = ep.Name() + r.ID = ep.ID() + r.Network = ep.Network() + } + return r +} + +func buildContainerResource(ci libnetwork.ContainerInfo) *containerResource { + r := &containerResource{} + if ci != nil { + r.ID = ci.ID() + } + return r +} + +/**************** + Options Parsers +*****************/ + +func (nc *networkCreate) parseOptions() []libnetwork.NetworkOption { + var setFctList []libnetwork.NetworkOption + + if nc.Options != nil { + setFctList = append(setFctList, libnetwork.NetworkOptionGeneric(nc.Options)) + } + + return setFctList +} + +func (ej *endpointJoin) parseOptions() []libnetwork.EndpointOption { + var setFctList []libnetwork.EndpointOption + if ej.HostName != "" { + setFctList = append(setFctList, libnetwork.JoinOptionHostname(ej.HostName)) + } + if ej.DomainName != "" { + setFctList = append(setFctList, libnetwork.JoinOptionDomainname(ej.DomainName)) + } + if ej.HostsPath != "" { + setFctList = append(setFctList, libnetwork.JoinOptionHostsPath(ej.HostsPath)) + } + if ej.ResolvConfPath != "" { + setFctList = append(setFctList, libnetwork.JoinOptionResolvConfPath(ej.ResolvConfPath)) + } + if ej.UseDefaultSandbox { + setFctList = append(setFctList, libnetwork.JoinOptionUseDefaultSandbox()) + } + if ej.DNS != nil { + for _, d := range ej.DNS { + setFctList = append(setFctList, libnetwork.JoinOptionDNS(d)) + } + } + if ej.ExtraHosts != nil { + for _, e := range ej.ExtraHosts { + setFctList = append(setFctList, libnetwork.JoinOptionExtraHost(e.Name, e.Address)) + } + } + if ej.ParentUpdates != nil { + for _, p := range ej.ParentUpdates { + setFctList = append(setFctList, libnetwork.JoinOptionParentUpdate(p.EndpointID, p.Name, p.Address)) + } + } + return setFctList +} + +/****************** + Process functions +*******************/ + +func processCreateDefaults(c libnetwork.NetworkController, nc *networkCreate) { + if nc.NetworkType == "" { + nc.NetworkType = c.Config().Daemon.DefaultDriver + } + if nc.NetworkType == BridgeNetworkDriver { + if nc.Options == nil { + nc.Options = make(map[string]interface{}) + } + genericData, ok := nc.Options[netlabel.GenericData] + if !ok { + genericData = make(map[string]interface{}) + } + gData := genericData.(map[string]interface{}) + + if _, ok := gData["BridgeName"]; !ok { + gData["BridgeName"] = nc.Name + } + if _, ok := gData["AllowNonDefaultBridge"]; !ok { + gData["AllowNonDefaultBridge"] = "true" + } + nc.Options[netlabel.GenericData] = genericData + } +} + +/*************************** + NetworkController interface +****************************/ +func procCreateNetwork(c libnetwork.NetworkController, vars map[string]string, body []byte) (interface{}, *responseStatus) { + var create networkCreate + + err := json.Unmarshal(body, &create) + if err != nil { + return "", &responseStatus{Status: "Invalid body: " + err.Error(), StatusCode: http.StatusBadRequest} + } + processCreateDefaults(c, &create) + + nw, err := c.NewNetwork(create.NetworkType, create.Name, create.parseOptions()...) + if err != nil { + return "", convertNetworkError(err) + } + + return nw.ID(), &createdResponse +} + +func procGetNetwork(c libnetwork.NetworkController, vars map[string]string, body []byte) (interface{}, *responseStatus) { + t, by := detectNetworkTarget(vars) + nw, errRsp := findNetwork(c, t, by) + if !errRsp.isOK() { + return nil, errRsp + } + return buildNetworkResource(nw), &successResponse +} + +func procGetNetworks(c libnetwork.NetworkController, vars map[string]string, body []byte) (interface{}, *responseStatus) { + var list []*networkResource + + // Look for query filters and validate + name, queryByName := vars[urlNwName] + shortID, queryByPid := vars[urlNwPID] + if queryByName && queryByPid { + return nil, &badQueryResponse + } + + if queryByName { + if nw, errRsp := findNetwork(c, name, byName); errRsp.isOK() { + list = append(list, buildNetworkResource(nw)) + } + } else if queryByPid { + // Return all the prefix-matching networks + l := func(nw libnetwork.Network) bool { + if strings.HasPrefix(nw.ID(), shortID) { + list = append(list, buildNetworkResource(nw)) + } + return false + } + c.WalkNetworks(l) + } else { + for _, nw := range c.Networks() { + list = append(list, buildNetworkResource(nw)) + } + } + + return list, &successResponse +} + +/****************** + Network interface +*******************/ +func procCreateEndpoint(c libnetwork.NetworkController, vars map[string]string, body []byte) (interface{}, *responseStatus) { + var ec endpointCreate + + err := json.Unmarshal(body, &ec) + if err != nil { + return "", &responseStatus{Status: "Invalid body: " + err.Error(), StatusCode: http.StatusBadRequest} + } + + nwT, nwBy := detectNetworkTarget(vars) + n, errRsp := findNetwork(c, nwT, nwBy) + if !errRsp.isOK() { + return "", errRsp + } + + var setFctList []libnetwork.EndpointOption + if ec.ExposedPorts != nil { + setFctList = append(setFctList, libnetwork.CreateOptionExposedPorts(ec.ExposedPorts)) + } + if ec.PortMapping != nil { + setFctList = append(setFctList, libnetwork.CreateOptionPortMapping(ec.PortMapping)) + } + + ep, err := n.CreateEndpoint(ec.Name, setFctList...) + if err != nil { + return "", convertNetworkError(err) + } + + return ep.ID(), &createdResponse +} + +func procGetEndpoint(c libnetwork.NetworkController, vars map[string]string, body []byte) (interface{}, *responseStatus) { + nwT, nwBy := detectNetworkTarget(vars) + epT, epBy := detectEndpointTarget(vars) + + ep, errRsp := findEndpoint(c, nwT, epT, nwBy, epBy) + if !errRsp.isOK() { + return nil, errRsp + } + + return buildEndpointResource(ep), &successResponse +} + +func procGetEndpoints(c libnetwork.NetworkController, vars map[string]string, body []byte) (interface{}, *responseStatus) { + // Look for query filters and validate + name, queryByName := vars[urlEpName] + shortID, queryByPid := vars[urlEpPID] + if queryByName && queryByPid { + return nil, &badQueryResponse + } + + nwT, nwBy := detectNetworkTarget(vars) + nw, errRsp := findNetwork(c, nwT, nwBy) + if !errRsp.isOK() { + return nil, errRsp + } + + var list []*endpointResource + + // If query parameter is specified, return a filtered collection + if queryByName { + if ep, errRsp := findEndpoint(c, nwT, name, nwBy, byName); errRsp.isOK() { + list = append(list, buildEndpointResource(ep)) + } + } else if queryByPid { + // Return all the prefix-matching endpoints + l := func(ep libnetwork.Endpoint) bool { + if strings.HasPrefix(ep.ID(), shortID) { + list = append(list, buildEndpointResource(ep)) + } + return false + } + nw.WalkEndpoints(l) + } else { + for _, ep := range nw.Endpoints() { + epr := buildEndpointResource(ep) + list = append(list, epr) + } + } + + return list, &successResponse +} + +func procDeleteNetwork(c libnetwork.NetworkController, vars map[string]string, body []byte) (interface{}, *responseStatus) { + target, by := detectNetworkTarget(vars) + + nw, errRsp := findNetwork(c, target, by) + if !errRsp.isOK() { + return nil, errRsp + } + + err := nw.Delete() + if err != nil { + return nil, convertNetworkError(err) + } + + return nil, &successResponse +} + +/****************** + Endpoint interface +*******************/ +func procJoinEndpoint(c libnetwork.NetworkController, vars map[string]string, body []byte) (interface{}, *responseStatus) { + var ej endpointJoin + err := json.Unmarshal(body, &ej) + if err != nil { + return nil, &responseStatus{Status: "Invalid body: " + err.Error(), StatusCode: http.StatusBadRequest} + } + + nwT, nwBy := detectNetworkTarget(vars) + epT, epBy := detectEndpointTarget(vars) + + ep, errRsp := findEndpoint(c, nwT, epT, nwBy, epBy) + if !errRsp.isOK() { + return nil, errRsp + } + + err = ep.Join(ej.ContainerID, ej.parseOptions()...) + if err != nil { + return nil, convertNetworkError(err) + } + return ep.Info().SandboxKey(), &successResponse +} + +func procLeaveEndpoint(c libnetwork.NetworkController, vars map[string]string, body []byte) (interface{}, *responseStatus) { + nwT, nwBy := detectNetworkTarget(vars) + epT, epBy := detectEndpointTarget(vars) + + ep, errRsp := findEndpoint(c, nwT, epT, nwBy, epBy) + if !errRsp.isOK() { + return nil, errRsp + } + + err := ep.Leave(vars[urlCnID]) + if err != nil { + return nil, convertNetworkError(err) + } + + return nil, &successResponse +} + +func procDeleteEndpoint(c libnetwork.NetworkController, vars map[string]string, body []byte) (interface{}, *responseStatus) { + nwT, nwBy := detectNetworkTarget(vars) + epT, epBy := detectEndpointTarget(vars) + + ep, errRsp := findEndpoint(c, nwT, epT, nwBy, epBy) + if !errRsp.isOK() { + return nil, errRsp + } + + err := ep.Delete() + if err != nil { + return nil, convertNetworkError(err) + } + + return nil, &successResponse +} + +/****************** + Service interface +*******************/ +func procGetServices(c libnetwork.NetworkController, vars map[string]string, body []byte) (interface{}, *responseStatus) { + // Look for query filters and validate + nwName, filterByNwName := vars[urlNwName] + svName, queryBySvName := vars[urlEpName] + shortID, queryBySvPID := vars[urlEpPID] + + if filterByNwName && queryBySvName || filterByNwName && queryBySvPID || queryBySvName && queryBySvPID { + return nil, &badQueryResponse + } + + var list []*endpointResource + + switch { + case filterByNwName: + // return all service present on the specified network + nw, errRsp := findNetwork(c, nwName, byName) + if !errRsp.isOK() { + return list, &successResponse + } + for _, ep := range nw.Endpoints() { + epr := buildEndpointResource(ep) + list = append(list, epr) + } + case queryBySvName: + // Look in each network for the service with the specified name + l := func(ep libnetwork.Endpoint) bool { + if ep.Name() == svName { + list = append(list, buildEndpointResource(ep)) + return true + } + return false + } + for _, nw := range c.Networks() { + nw.WalkEndpoints(l) + } + case queryBySvPID: + // Return all the prefix-matching services + l := func(ep libnetwork.Endpoint) bool { + if strings.HasPrefix(ep.ID(), shortID) { + list = append(list, buildEndpointResource(ep)) + } + return false + } + for _, nw := range c.Networks() { + nw.WalkEndpoints(l) + } + default: + for _, nw := range c.Networks() { + for _, ep := range nw.Endpoints() { + epr := buildEndpointResource(ep) + list = append(list, epr) + } + } + } + + return list, &successResponse +} + +func procGetService(c libnetwork.NetworkController, vars map[string]string, body []byte) (interface{}, *responseStatus) { + epT, epBy := detectEndpointTarget(vars) + sv, errRsp := findService(c, epT, epBy) + if !errRsp.isOK() { + return nil, endpointToService(errRsp) + } + return buildEndpointResource(sv), &successResponse +} + +func procGetContainers(c libnetwork.NetworkController, vars map[string]string, body []byte) (interface{}, *responseStatus) { + epT, epBy := detectEndpointTarget(vars) + sv, errRsp := findService(c, epT, epBy) + if !errRsp.isOK() { + return nil, endpointToService(errRsp) + } + var list []*containerResource + if sv.ContainerInfo() != nil { + list = append(list, buildContainerResource(sv.ContainerInfo())) + } + return list, &successResponse +} + +func procPublishService(c libnetwork.NetworkController, vars map[string]string, body []byte) (interface{}, *responseStatus) { + var sp servicePublish + + err := json.Unmarshal(body, &sp) + if err != nil { + return "", &responseStatus{Status: "Invalid body: " + err.Error(), StatusCode: http.StatusBadRequest} + } + + n, errRsp := findNetwork(c, sp.Network, byName) + if !errRsp.isOK() { + return "", errRsp + } + + var setFctList []libnetwork.EndpointOption + if sp.ExposedPorts != nil { + setFctList = append(setFctList, libnetwork.CreateOptionExposedPorts(sp.ExposedPorts)) + } + if sp.PortMapping != nil { + setFctList = append(setFctList, libnetwork.CreateOptionPortMapping(sp.PortMapping)) + } + + ep, err := n.CreateEndpoint(sp.Name, setFctList...) + if err != nil { + return "", endpointToService(convertNetworkError(err)) + } + + return ep.ID(), &createdResponse +} + +func procUnpublishService(c libnetwork.NetworkController, vars map[string]string, body []byte) (interface{}, *responseStatus) { + epT, epBy := detectEndpointTarget(vars) + sv, errRsp := findService(c, epT, epBy) + if !errRsp.isOK() { + return nil, errRsp + } + err := sv.Delete() + if err != nil { + return nil, endpointToService(convertNetworkError(err)) + } + return nil, &successResponse +} + +func procAttachBackend(c libnetwork.NetworkController, vars map[string]string, body []byte) (interface{}, *responseStatus) { + var bk endpointJoin + err := json.Unmarshal(body, &bk) + if err != nil { + return nil, &responseStatus{Status: "Invalid body: " + err.Error(), StatusCode: http.StatusBadRequest} + } + + epT, epBy := detectEndpointTarget(vars) + sv, errRsp := findService(c, epT, epBy) + if !errRsp.isOK() { + return nil, errRsp + } + + err = sv.Join(bk.ContainerID, bk.parseOptions()...) + if err != nil { + return nil, convertNetworkError(err) + } + return sv.Info().SandboxKey(), &successResponse +} + +func procDetachBackend(c libnetwork.NetworkController, vars map[string]string, body []byte) (interface{}, *responseStatus) { + epT, epBy := detectEndpointTarget(vars) + sv, errRsp := findService(c, epT, epBy) + if !errRsp.isOK() { + return nil, errRsp + } + + err := sv.Leave(vars[urlCnID]) + if err != nil { + return nil, convertNetworkError(err) + } + + return nil, &successResponse +} + +/*********** + Utilities +************/ +const ( + byID = iota + byName +) + +func detectNetworkTarget(vars map[string]string) (string, int) { + if target, ok := vars[urlNwName]; ok { + return target, byName + } + if target, ok := vars[urlNwID]; ok { + return target, byID + } + // vars are populated from the URL, following cannot happen + panic("Missing URL variable parameter for network") +} + +func detectEndpointTarget(vars map[string]string) (string, int) { + if target, ok := vars[urlEpName]; ok { + return target, byName + } + if target, ok := vars[urlEpID]; ok { + return target, byID + } + // vars are populated from the URL, following cannot happen + panic("Missing URL variable parameter for endpoint") +} + +func findNetwork(c libnetwork.NetworkController, s string, by int) (libnetwork.Network, *responseStatus) { + var ( + nw libnetwork.Network + err error + ) + switch by { + case byID: + nw, err = c.NetworkByID(s) + case byName: + if s == "" { + s = c.Config().Daemon.DefaultNetwork + } + nw, err = c.NetworkByName(s) + default: + panic(fmt.Sprintf("unexpected selector for network search: %d", by)) + } + if err != nil { + if _, ok := err.(types.NotFoundError); ok { + return nil, &responseStatus{Status: "Resource not found: Network", StatusCode: http.StatusNotFound} + } + return nil, &responseStatus{Status: err.Error(), StatusCode: http.StatusBadRequest} + } + return nw, &successResponse +} + +func findEndpoint(c libnetwork.NetworkController, ns, es string, nwBy, epBy int) (libnetwork.Endpoint, *responseStatus) { + nw, errRsp := findNetwork(c, ns, nwBy) + if !errRsp.isOK() { + return nil, errRsp + } + var ( + err error + ep libnetwork.Endpoint + ) + switch epBy { + case byID: + ep, err = nw.EndpointByID(es) + case byName: + ep, err = nw.EndpointByName(es) + default: + panic(fmt.Sprintf("unexpected selector for endpoint search: %d", epBy)) + } + if err != nil { + if _, ok := err.(types.NotFoundError); ok { + return nil, &responseStatus{Status: "Resource not found: Endpoint", StatusCode: http.StatusNotFound} + } + return nil, &responseStatus{Status: err.Error(), StatusCode: http.StatusBadRequest} + } + return ep, &successResponse +} + +func findService(c libnetwork.NetworkController, svs string, svBy int) (libnetwork.Endpoint, *responseStatus) { + for _, nw := range c.Networks() { + var ( + ep libnetwork.Endpoint + err error + ) + switch svBy { + case byID: + ep, err = nw.EndpointByID(svs) + case byName: + ep, err = nw.EndpointByName(svs) + default: + panic(fmt.Sprintf("unexpected selector for service search: %d", svBy)) + } + if err == nil { + return ep, &successResponse + } else if _, ok := err.(types.NotFoundError); !ok { + return nil, convertNetworkError(err) + } + } + return nil, &responseStatus{Status: "Service not found", StatusCode: http.StatusNotFound} +} + +func endpointToService(rsp *responseStatus) *responseStatus { + rsp.Status = strings.Replace(rsp.Status, "endpoint", "service", -1) + return rsp +} + +func convertNetworkError(err error) *responseStatus { + var code int + switch err.(type) { + case types.BadRequestError: + code = http.StatusBadRequest + case types.ForbiddenError: + code = http.StatusForbidden + case types.NotFoundError: + code = http.StatusNotFound + case types.TimeoutError: + code = http.StatusRequestTimeout + case types.NotImplementedError: + code = http.StatusNotImplemented + case types.NoServiceError: + code = http.StatusServiceUnavailable + case types.InternalError: + code = http.StatusInternalServerError + default: + code = http.StatusInternalServerError + } + return &responseStatus{Status: err.Error(), StatusCode: code} +} + +func writeJSON(w http.ResponseWriter, code int, v interface{}) error { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(code) + return json.NewEncoder(w).Encode(v) +} diff --git a/Godeps/_workspace/src/github.com/docker/libnetwork/api/api_test.go b/Godeps/_workspace/src/github.com/docker/libnetwork/api/api_test.go new file mode 100644 index 000000000..8f1c48f99 --- /dev/null +++ b/Godeps/_workspace/src/github.com/docker/libnetwork/api/api_test.go @@ -0,0 +1,2183 @@ +package api + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "os" + "regexp" + "runtime" + "testing" + + "github.com/docker/docker/pkg/reexec" + "github.com/docker/libnetwork" + "github.com/docker/libnetwork/netlabel" + "github.com/docker/libnetwork/netutils" + "github.com/docker/libnetwork/options" + "github.com/docker/libnetwork/types" +) + +const ( + bridgeNetType = "bridge" + bridgeName = "docker0" +) + +func getEmptyGenericOption() map[string]interface{} { + genericOption := make(map[string]interface{}) + genericOption[netlabel.GenericData] = options.Generic{} + return genericOption +} + +func i2s(i interface{}) string { + s, ok := i.(string) + if !ok { + panic(fmt.Sprintf("Failed i2s for %v", i)) + } + return s +} + +func i2e(i interface{}) *endpointResource { + s, ok := i.(*endpointResource) + if !ok { + panic(fmt.Sprintf("Failed i2e for %v", i)) + } + return s +} + +func i2eL(i interface{}) []*endpointResource { + s, ok := i.([]*endpointResource) + if !ok { + panic(fmt.Sprintf("Failed i2eL for %v", i)) + } + return s +} + +func i2n(i interface{}) *networkResource { + s, ok := i.(*networkResource) + if !ok { + panic(fmt.Sprintf("Failed i2n for %v", i)) + } + return s +} + +func i2nL(i interface{}) []*networkResource { + s, ok := i.([]*networkResource) + if !ok { + panic(fmt.Sprintf("Failed i2nL for %v", i)) + } + return s +} + +func i2cL(i interface{}) []*containerResource { + s, ok := i.([]*containerResource) + if !ok { + panic(fmt.Sprintf("Failed i2cL for %v", i)) + } + return s +} + +func createTestNetwork(t *testing.T, network string) (libnetwork.NetworkController, libnetwork.Network) { + c, err := libnetwork.New() + if err != nil { + t.Fatal(err) + } + + err = c.ConfigureNetworkDriver(bridgeNetType, nil) + if err != nil { + t.Fatal(err) + } + + netOption := options.Generic{ + netlabel.GenericData: options.Generic{ + "BridgeName": network, + "AllowNonDefaultBridge": true, + }, + } + netGeneric := libnetwork.NetworkOptionGeneric(netOption) + nw, err := c.NewNetwork(bridgeNetType, network, netGeneric) + if err != nil { + t.Fatal(err) + } + + return c, nw +} + +func TestMain(m *testing.M) { + if reexec.Init() { + return + } + os.Exit(m.Run()) +} + +func TestJoinOptionParser(t *testing.T) { + hn := "host1" + dn := "docker.com" + hp := "/etc/hosts" + rc := "/etc/resolv.conf" + dnss := []string{"8.8.8.8", "172.28.34.5"} + ehs := []endpointExtraHost{endpointExtraHost{Name: "extra1", Address: "172.28.9.1"}, endpointExtraHost{Name: "extra2", Address: "172.28.9.2"}} + pus := []endpointParentUpdate{endpointParentUpdate{EndpointID: "abc123def456", Name: "serv1", Address: "172.28.30.123"}} + + ej := endpointJoin{ + HostName: hn, + DomainName: dn, + HostsPath: hp, + ResolvConfPath: rc, + DNS: dnss, + ExtraHosts: ehs, + ParentUpdates: pus, + UseDefaultSandbox: true, + } + + if len(ej.parseOptions()) != 10 { + t.Fatalf("Failed to generate all libnetwork.EndpointJoinOption methods libnetwork.EndpointJoinOption method") + } + +} + +func TestJson(t *testing.T) { + nc := networkCreate{NetworkType: bridgeNetType} + b, err := json.Marshal(nc) + if err != nil { + t.Fatal(err) + } + + var ncp networkCreate + err = json.Unmarshal(b, &ncp) + if err != nil { + t.Fatal(err) + } + + if nc.NetworkType != ncp.NetworkType { + t.Fatalf("Incorrect networkCreate after json encoding/deconding: %v", ncp) + } + + jl := endpointJoin{ContainerID: "abcdef456789"} + b, err = json.Marshal(jl) + if err != nil { + t.Fatal(err) + } + + var jld endpointJoin + err = json.Unmarshal(b, &jld) + if err != nil { + t.Fatal(err) + } + + if jl.ContainerID != jld.ContainerID { + t.Fatalf("Incorrect endpointJoin after json encoding/deconding: %v", jld) + } +} + +func TestCreateDeleteNetwork(t *testing.T) { + defer netutils.SetupTestNetNS(t)() + + c, err := libnetwork.New() + if err != nil { + t.Fatal(err) + } + err = c.ConfigureNetworkDriver(bridgeNetType, nil) + if err != nil { + t.Fatal(err) + } + + badBody, err := json.Marshal("bad body") + if err != nil { + t.Fatal(err) + } + + vars := make(map[string]string) + _, errRsp := procCreateNetwork(c, nil, badBody) + if errRsp == &createdResponse { + t.Fatalf("Expected to fail but succeeded") + } + if errRsp.StatusCode != http.StatusBadRequest { + t.Fatalf("Expected StatusBadRequest status code, got: %v", errRsp) + } + + incompleteBody, err := json.Marshal(networkCreate{}) + if err != nil { + t.Fatal(err) + } + + _, errRsp = procCreateNetwork(c, vars, incompleteBody) + if errRsp == &createdResponse { + t.Fatalf("Expected to fail but succeeded") + } + if errRsp.StatusCode != http.StatusBadRequest { + t.Fatalf("Expected StatusBadRequest status code, got: %v", errRsp) + } + + ops := options.Generic{ + netlabel.EnableIPv6: true, + netlabel.GenericData: map[string]string{ + "BridgeName": "abc", + "AllowNonDefaultBridge": "true", + "FixedCIDRv6": "fe80::1/64", + "AddressIP": "172.28.30.254/24", + }, + } + nc := networkCreate{Name: "network_1", NetworkType: bridgeNetType, Options: ops} + goodBody, err := json.Marshal(nc) + if err != nil { + t.Fatal(err) + } + + _, errRsp = procCreateNetwork(c, vars, goodBody) + if errRsp != &createdResponse { + t.Fatalf("Unexepected failure: %v", errRsp) + } + + vars[urlNwName] = "" + _, errRsp = procDeleteNetwork(c, vars, nil) + if errRsp == &successResponse { + t.Fatalf("Expected to fail but succeeded") + } + + vars[urlNwName] = "abc" + _, errRsp = procDeleteNetwork(c, vars, nil) + if errRsp == &successResponse { + t.Fatalf("Expected to fail but succeeded") + } + + vars[urlNwName] = "network_1" + _, errRsp = procDeleteNetwork(c, vars, nil) + if errRsp != &successResponse { + t.Fatalf("Unexepected failure: %v", errRsp) + } +} + +func TestGetNetworksAndEndpoints(t *testing.T) { + defer netutils.SetupTestNetNS(t)() + + c, err := libnetwork.New() + if err != nil { + t.Fatal(err) + } + err = c.ConfigureNetworkDriver(bridgeNetType, nil) + if err != nil { + t.Fatal(err) + } + + ops := options.Generic{ + netlabel.GenericData: map[string]string{ + "BridgeName": "api_test_nw", + "AllowNonDefaultBridge": "true", + }, + } + + nc := networkCreate{Name: "sh", NetworkType: bridgeNetType, Options: ops} + body, err := json.Marshal(nc) + if err != nil { + t.Fatal(err) + } + + vars := make(map[string]string) + inid, errRsp := procCreateNetwork(c, vars, body) + if errRsp != &createdResponse { + t.Fatalf("Unexepected failure: %v", errRsp) + } + nid, ok := inid.(string) + if !ok { + t.FailNow() + } + + ec1 := endpointCreate{ + Name: "ep1", + ExposedPorts: []types.TransportPort{ + types.TransportPort{Proto: types.TCP, Port: uint16(5000)}, + types.TransportPort{Proto: types.UDP, Port: uint16(400)}, + types.TransportPort{Proto: types.TCP, Port: uint16(600)}, + }, + PortMapping: []types.PortBinding{ + types.PortBinding{Proto: types.TCP, Port: uint16(230), HostPort: uint16(23000)}, + types.PortBinding{Proto: types.UDP, Port: uint16(200), HostPort: uint16(22000)}, + types.PortBinding{Proto: types.TCP, Port: uint16(120), HostPort: uint16(12000)}, + }, + } + b1, err := json.Marshal(ec1) + if err != nil { + t.Fatal(err) + } + ec2 := endpointCreate{Name: "ep2"} + b2, err := json.Marshal(ec2) + if err != nil { + t.Fatal(err) + } + + vars[urlNwName] = "sh" + vars[urlEpName] = "ep1" + ieid1, errRsp := procCreateEndpoint(c, vars, b1) + if errRsp != &createdResponse { + t.Fatalf("Unexepected failure: %v", errRsp) + } + eid1 := i2s(ieid1) + vars[urlEpName] = "ep2" + ieid2, errRsp := procCreateEndpoint(c, vars, b2) + if errRsp != &createdResponse { + t.Fatalf("Unexepected failure: %v", errRsp) + } + eid2 := i2s(ieid2) + + vars[urlNwName] = "" + vars[urlEpName] = "ep1" + _, errRsp = procGetEndpoint(c, vars, nil) + if errRsp == &successResponse { + t.Fatalf("Expected failure but succeeded: %v", errRsp) + } + if errRsp.StatusCode != http.StatusBadRequest { + t.Fatalf("Expected to fail with http.StatusBadRequest, but got: %d", errRsp.StatusCode) + } + + vars = make(map[string]string) + vars[urlNwName] = "sh" + vars[urlEpID] = "" + _, errRsp = procGetEndpoint(c, vars, nil) + if errRsp == &successResponse { + t.Fatalf("Expected failure but succeeded: %v", errRsp) + } + if errRsp.StatusCode != http.StatusBadRequest { + t.Fatalf("Expected to fail with http.StatusBadRequest, but got: %d", errRsp.StatusCode) + } + + vars = make(map[string]string) + vars[urlNwID] = "" + vars[urlEpID] = eid1 + _, errRsp = procGetEndpoint(c, vars, nil) + if errRsp == &successResponse { + t.Fatalf("Expected failure but succeeded: %v", errRsp) + } + if errRsp.StatusCode != http.StatusBadRequest { + t.Fatalf("Expected to fail with http.StatusBadRequest, but got: %d", errRsp.StatusCode) + } + + // nw by name and ep by id + vars[urlNwName] = "sh" + i1, errRsp := procGetEndpoint(c, vars, nil) + if errRsp != &successResponse { + t.Fatalf("Unexepected failure: %v", errRsp) + } + // nw by name and ep by name + delete(vars, urlEpID) + vars[urlEpName] = "ep1" + i2, errRsp := procGetEndpoint(c, vars, nil) + if errRsp != &successResponse { + t.Fatalf("Unexepected failure: %v", errRsp) + } + // nw by id and ep by name + delete(vars, urlNwName) + vars[urlNwID] = nid + i3, errRsp := procGetEndpoint(c, vars, nil) + if errRsp != &successResponse { + t.Fatalf("Unexepected failure: %v", errRsp) + } + // nw by id and ep by id + delete(vars, urlEpName) + vars[urlEpID] = eid1 + i4, errRsp := procGetEndpoint(c, vars, nil) + if errRsp != &successResponse { + t.Fatalf("Unexepected failure: %v", errRsp) + } + + id1 := i2e(i1).ID + if id1 != i2e(i2).ID || id1 != i2e(i3).ID || id1 != i2e(i4).ID { + t.Fatalf("Endpoints retireved via different query parameters differ: %v, %v, %v, %v", i1, i2, i3, i4) + } + + vars[urlNwName] = "" + _, errRsp = procGetEndpoints(c, vars, nil) + if errRsp == &successResponse { + t.Fatalf("Expected failure, got: %v", errRsp) + } + + delete(vars, urlNwName) + vars[urlNwID] = "fakeID" + _, errRsp = procGetEndpoints(c, vars, nil) + if errRsp == &successResponse { + t.Fatalf("Expected failure, got: %v", errRsp) + } + + vars[urlNwID] = nid + _, errRsp = procGetEndpoints(c, vars, nil) + if errRsp != &successResponse { + t.Fatalf("Unexepected failure: %v", errRsp) + } + + vars[urlNwName] = "sh" + iepList, errRsp := procGetEndpoints(c, vars, nil) + if errRsp != &successResponse { + t.Fatalf("Unexepected failure: %v", errRsp) + } + epList := i2eL(iepList) + if len(epList) != 2 { + t.Fatalf("Did not return the expected number (2) of endpoint resources: %d", len(epList)) + } + if "sh" != epList[0].Network || "sh" != epList[1].Network { + t.Fatalf("Did not find expected network name in endpoint resources") + } + + vars = make(map[string]string) + vars[urlNwName] = "" + _, errRsp = procGetNetwork(c, vars, nil) + if errRsp == &successResponse { + t.Fatalf("Exepected failure, got: %v", errRsp) + } + vars[urlNwName] = "shhhhh" + _, errRsp = procGetNetwork(c, vars, nil) + if errRsp == &successResponse { + t.Fatalf("Exepected failure, got: %v", errRsp) + } + vars[urlNwName] = "sh" + inr1, errRsp := procGetNetwork(c, vars, nil) + if errRsp != &successResponse { + t.Fatalf("Unexepected failure: %v", errRsp) + } + nr1 := i2n(inr1) + + delete(vars, urlNwName) + vars[urlNwID] = "cacca" + _, errRsp = procGetNetwork(c, vars, nil) + if errRsp == &successResponse { + t.Fatalf("Unexepected failure: %v", errRsp) + } + vars[urlNwID] = nid + inr2, errRsp := procGetNetwork(c, vars, nil) + if errRsp != &successResponse { + t.Fatalf("procgetNetworkByName() != procgetNetworkById(), %v vs %v", inr1, inr2) + } + nr2 := i2n(inr2) + if nr1.Name != nr2.Name || nr1.Type != nr2.Type || nr1.ID != nr2.ID || len(nr1.Endpoints) != len(nr2.Endpoints) { + t.Fatalf("Get by name and Get failure: %v", errRsp) + } + + if len(nr1.Endpoints) != 2 { + t.Fatalf("Did not find the expected number (2) of endpoint resources in the network resource: %d", len(nr1.Endpoints)) + } + for _, er := range nr1.Endpoints { + if er.ID != eid1 && er.ID != eid2 { + t.Fatalf("Did not find the expected endpoint resources in the network resource: %v", nr1.Endpoints) + } + } + + iList, errRsp := procGetNetworks(c, nil, nil) + if errRsp != &successResponse { + t.Fatalf("Unexepected failure: %v", errRsp) + } + netList := i2nL(iList) + if len(netList) != 1 { + t.Fatalf("Did not return the expected number of network resources") + } + if nid != netList[0].ID { + t.Fatalf("Did not find expected network %s: %v", nid, netList) + } + + _, errRsp = procDeleteNetwork(c, vars, nil) + if errRsp == &successResponse { + t.Fatalf("Exepected failure, got: %v", errRsp) + } + + vars[urlEpName] = "ep1" + _, errRsp = procDeleteEndpoint(c, vars, nil) + if errRsp != &successResponse { + t.Fatalf("Unexepected failure: %v", errRsp) + } + delete(vars, urlEpName) + iepList, errRsp = procGetEndpoints(c, vars, nil) + if errRsp != &successResponse { + t.Fatalf("Unexepected failure: %v", errRsp) + } + epList = i2eL(iepList) + if len(epList) != 1 { + t.Fatalf("Did not return the expected number (1) of endpoint resources: %d", len(epList)) + } + + vars[urlEpName] = "ep2" + _, errRsp = procDeleteEndpoint(c, vars, nil) + if errRsp != &successResponse { + t.Fatalf("Unexepected failure: %v", errRsp) + } + iepList, errRsp = procGetEndpoints(c, vars, nil) + if errRsp != &successResponse { + t.Fatalf("Unexepected failure: %v", errRsp) + } + epList = i2eL(iepList) + if len(epList) != 0 { + t.Fatalf("Did not return the expected number (0) of endpoint resources: %d", len(epList)) + } + + _, errRsp = procDeleteNetwork(c, vars, nil) + if errRsp != &successResponse { + t.Fatalf("Unexepected failure: %v", errRsp) + } + + iList, errRsp = procGetNetworks(c, nil, nil) + if errRsp != &successResponse { + t.Fatalf("Unexepected failure: %v", errRsp) + } + netList = i2nL(iList) + if len(netList) != 0 { + t.Fatalf("Did not return the expected number of network resources") + } +} + +func TestProcGetServices(t *testing.T) { + defer netutils.SetupTestNetNS(t)() + + c, err := libnetwork.New() + if err != nil { + t.Fatal(err) + } + + err = c.ConfigureNetworkDriver(bridgeNetType, nil) + if err != nil { + t.Fatal(err) + } + + // Create 2 networks + netName1 := "production" + netOption := options.Generic{ + netlabel.GenericData: options.Generic{ + "BridgeName": netName1, + "AllowNonDefaultBridge": true, + }, + } + nw1, err := c.NewNetwork(bridgeNetType, netName1, libnetwork.NetworkOptionGeneric(netOption)) + if err != nil { + t.Fatal(err) + } + + netName2 := "work-dev" + netOption = options.Generic{ + netlabel.GenericData: options.Generic{ + "BridgeName": netName2, + "AllowNonDefaultBridge": true, + }, + } + nw2, err := c.NewNetwork(bridgeNetType, netName2, libnetwork.NetworkOptionGeneric(netOption)) + if err != nil { + t.Fatal(err) + } + + vars := make(map[string]string) + li, errRsp := procGetServices(c, vars, nil) + if !errRsp.isOK() { + t.Fatalf("Unexpected failure: %v", errRsp) + } + list := i2eL(li) + if len(list) != 0 { + t.Fatalf("Unexpected services in response: %v", list) + } + + // Add a couple of services on one network and one on the other network + ep11, err := nw1.CreateEndpoint("db-prod") + if err != nil { + t.Fatal(err) + } + ep12, err := nw1.CreateEndpoint("web-prod") + if err != nil { + t.Fatal(err) + } + ep21, err := nw2.CreateEndpoint("db-dev") + if err != nil { + t.Fatal(err) + } + + li, errRsp = procGetServices(c, vars, nil) + if !errRsp.isOK() { + t.Fatalf("Unexpected failure: %v", errRsp) + } + list = i2eL(li) + if len(list) != 3 { + t.Fatalf("Unexpected services in response: %v", list) + } + + // Filter by network + vars[urlNwName] = netName1 + li, errRsp = procGetServices(c, vars, nil) + if !errRsp.isOK() { + t.Fatalf("Unexpected failure: %v", errRsp) + } + list = i2eL(li) + if len(list) != 2 { + t.Fatalf("Unexpected services in response: %v", list) + } + + vars[urlNwName] = netName2 + li, errRsp = procGetServices(c, vars, nil) + if !errRsp.isOK() { + t.Fatalf("Unexpected failure: %v", errRsp) + } + list = i2eL(li) + if len(list) != 1 { + t.Fatalf("Unexpected services in response: %v", list) + } + + vars[urlNwName] = "unknown-network" + li, errRsp = procGetServices(c, vars, nil) + if !errRsp.isOK() { + t.Fatalf("Unexpected failure: %v", errRsp) + } + list = i2eL(li) + if len(list) != 0 { + t.Fatalf("Unexpected services in response: %v", list) + } + + // Query by name + delete(vars, urlNwName) + vars[urlEpName] = "db-prod" + li, errRsp = procGetServices(c, vars, nil) + if !errRsp.isOK() { + t.Fatalf("Unexpected failure: %v", errRsp) + } + list = i2eL(li) + if len(list) != 1 { + t.Fatalf("Unexpected services in response: %v", list) + } + + vars[urlEpName] = "no-service" + li, errRsp = procGetServices(c, vars, nil) + if !errRsp.isOK() { + t.Fatalf("Unexpected failure: %v", errRsp) + } + list = i2eL(li) + if len(list) != 0 { + t.Fatalf("Unexpected services in response: %v", list) + } + + // Query by id or partial id + delete(vars, urlEpName) + vars[urlEpPID] = ep12.ID() + li, errRsp = procGetServices(c, vars, nil) + if !errRsp.isOK() { + t.Fatalf("Unexpected failure: %v", errRsp) + } + list = i2eL(li) + if len(list) != 1 { + t.Fatalf("Unexpected services in response: %v", list) + } + if list[0].ID != ep12.ID() { + t.Fatalf("Unexpected element in response: %v", list) + } + + vars[urlEpPID] = "non-id" + li, errRsp = procGetServices(c, vars, nil) + if !errRsp.isOK() { + t.Fatalf("Unexpected failure: %v", errRsp) + } + list = i2eL(li) + if len(list) != 0 { + t.Fatalf("Unexpected services in response: %v", list) + } + + delete(vars, urlEpPID) + err = ep11.Delete() + if err != nil { + t.Fatal(err) + } + err = ep12.Delete() + if err != nil { + t.Fatal(err) + } + err = ep21.Delete() + if err != nil { + t.Fatal(err) + } + + li, errRsp = procGetServices(c, vars, nil) + if !errRsp.isOK() { + t.Fatalf("Unexpected failure: %v", errRsp) + } + list = i2eL(li) + if len(list) != 0 { + t.Fatalf("Unexpected services in response: %v", list) + } +} + +func TestProcGetService(t *testing.T) { + defer netutils.SetupTestNetNS(t)() + + c, nw := createTestNetwork(t, "network") + ep1, err := nw.CreateEndpoint("db") + if err != nil { + t.Fatal(err) + } + ep2, err := nw.CreateEndpoint("web") + if err != nil { + t.Fatal(err) + } + + vars := map[string]string{urlEpID: ""} + _, errRsp := procGetService(c, vars, nil) + if errRsp.isOK() { + t.Fatalf("Expected failure, but suceeded") + } + if errRsp.StatusCode != http.StatusBadRequest { + t.Fatalf("Expected %d, but got: %d", http.StatusBadRequest, errRsp.StatusCode) + } + + vars[urlEpID] = "unknown-service-id" + _, errRsp = procGetService(c, vars, nil) + if errRsp.isOK() { + t.Fatalf("Expected failure, but suceeded") + } + if errRsp.StatusCode != http.StatusNotFound { + t.Fatalf("Expected %d, but got: %d. (%v)", http.StatusNotFound, errRsp.StatusCode, errRsp) + } + + vars[urlEpID] = ep1.ID() + si, errRsp := procGetService(c, vars, nil) + if !errRsp.isOK() { + t.Fatalf("Unexpected failure: %v", errRsp) + } + sv := i2e(si) + if sv.ID != ep1.ID() { + t.Fatalf("Unexpected service resource returned: %v", sv) + } + + vars[urlEpID] = ep2.ID() + si, errRsp = procGetService(c, vars, nil) + if !errRsp.isOK() { + t.Fatalf("Unexpected failure: %v", errRsp) + } + sv = i2e(si) + if sv.ID != ep2.ID() { + t.Fatalf("Unexpected service resource returned: %v", sv) + } +} + +func TestProcPublishUnpublishService(t *testing.T) { + defer netutils.SetupTestNetNS(t)() + + c, _ := createTestNetwork(t, "network") + vars := make(map[string]string) + + vbad, err := json.Marshal("bad service create data") + if err != nil { + t.Fatal(err) + } + _, errRsp := procPublishService(c, vars, vbad) + if errRsp == &createdResponse { + t.Fatalf("Expected to fail but succeeded") + } + if errRsp.StatusCode != http.StatusBadRequest { + t.Fatalf("Expected %d. Got: %v", http.StatusBadRequest, errRsp) + } + + b, err := json.Marshal(servicePublish{Name: ""}) + if err != nil { + t.Fatal(err) + } + _, errRsp = procPublishService(c, vars, b) + if errRsp == &createdResponse { + t.Fatalf("Expected to fail but succeeded") + } + if errRsp.StatusCode != http.StatusBadRequest { + t.Fatalf("Expected %d. Got: %v", http.StatusBadRequest, errRsp) + } + + b, err = json.Marshal(servicePublish{Name: "db"}) + if err != nil { + t.Fatal(err) + } + _, errRsp = procPublishService(c, vars, b) + if errRsp == &createdResponse { + t.Fatalf("Expected to fail but succeeded") + } + if errRsp.StatusCode != http.StatusBadRequest { + t.Fatalf("Expected %d. Got: %v", http.StatusBadRequest, errRsp) + } + + b, err = json.Marshal(servicePublish{Name: "db", Network: "unknown-network"}) + if err != nil { + t.Fatal(err) + } + _, errRsp = procPublishService(c, vars, b) + if errRsp == &createdResponse { + t.Fatalf("Expected to fail but succeeded") + } + if errRsp.StatusCode != http.StatusNotFound { + t.Fatalf("Expected %d. Got: %v", http.StatusNotFound, errRsp) + } + + b, err = json.Marshal(servicePublish{Name: "", Network: "network"}) + if err != nil { + t.Fatal(err) + } + _, errRsp = procPublishService(c, vars, b) + if errRsp == &createdResponse { + t.Fatalf("Expected to fail but succeeded") + } + if errRsp.StatusCode != http.StatusBadRequest { + t.Fatalf("Expected %d. Got: %v", http.StatusBadRequest, errRsp) + } + + b, err = json.Marshal(servicePublish{Name: "db", Network: "network"}) + if err != nil { + t.Fatal(err) + } + _, errRsp = procPublishService(c, vars, b) + if errRsp != &createdResponse { + t.Fatalf("Unexpected failure: %v", errRsp) + } + + sp := servicePublish{ + Name: "web", + Network: "network", + ExposedPorts: []types.TransportPort{ + types.TransportPort{Proto: types.TCP, Port: uint16(6000)}, + types.TransportPort{Proto: types.UDP, Port: uint16(500)}, + types.TransportPort{Proto: types.TCP, Port: uint16(700)}, + }, + PortMapping: []types.PortBinding{ + types.PortBinding{Proto: types.TCP, Port: uint16(1230), HostPort: uint16(37000)}, + types.PortBinding{Proto: types.UDP, Port: uint16(1200), HostPort: uint16(36000)}, + types.PortBinding{Proto: types.TCP, Port: uint16(1120), HostPort: uint16(35000)}, + }, + } + b, err = json.Marshal(sp) + if err != nil { + t.Fatal(err) + } + si, errRsp := procPublishService(c, vars, b) + if errRsp != &createdResponse { + t.Fatalf("Unexpected failure: %v", errRsp) + } + sid := i2s(si) + + vars[urlEpID] = "" + _, errRsp = procUnpublishService(c, vars, nil) + if errRsp.isOK() { + t.Fatalf("Expected failure but succeeded") + } + if errRsp.StatusCode != http.StatusBadRequest { + t.Fatalf("Expected %d. Got: %v", http.StatusBadRequest, errRsp) + } + + vars[urlEpID] = "unknown-service-id" + _, errRsp = procUnpublishService(c, vars, nil) + if errRsp.isOK() { + t.Fatalf("Expected failure but succeeded") + } + if errRsp.StatusCode != http.StatusNotFound { + t.Fatalf("Expected %d. Got: %v", http.StatusNotFound, errRsp) + } + + vars[urlEpID] = sid + _, errRsp = procUnpublishService(c, vars, nil) + if !errRsp.isOK() { + t.Fatalf("Unexpected failure: %v", errRsp) + } + + _, errRsp = procGetService(c, vars, nil) + if errRsp.isOK() { + t.Fatalf("Expected failure, but suceeded") + } + if errRsp.StatusCode != http.StatusNotFound { + t.Fatalf("Expected %d, but got: %d. (%v)", http.StatusNotFound, errRsp.StatusCode, errRsp) + } +} + +func TestAttachDetachBackend(t *testing.T) { + defer netutils.SetupTestNetNS(t)() + + c, nw := createTestNetwork(t, "network") + ep1, err := nw.CreateEndpoint("db") + if err != nil { + t.Fatal(err) + } + + vars := make(map[string]string) + + vbad, err := json.Marshal("bad data") + if err != nil { + t.Fatal(err) + } + _, errRsp := procAttachBackend(c, vars, vbad) + if errRsp == &successResponse { + t.Fatalf("Expected failure, got: %v", errRsp) + } + + vars[urlEpName] = "endpoint" + bad, err := json.Marshal(endpointJoin{}) + if err != nil { + t.Fatal(err) + } + _, errRsp = procAttachBackend(c, vars, bad) + if errRsp == &successResponse { + t.Fatalf("Expected failure, got: %v", errRsp) + } + if errRsp.StatusCode != http.StatusNotFound { + t.Fatalf("Expected %d. Got: %v", http.StatusNotFound, errRsp) + } + + _, errRsp = procGetContainers(c, vars, nil) + if errRsp.isOK() { + t.Fatalf("Expected failure. Got %v", errRsp) + } + if errRsp.StatusCode != http.StatusNotFound { + t.Fatalf("Expected %d. Got: %v", http.StatusNotFound, errRsp) + } + + vars[urlEpName] = "db" + _, errRsp = procAttachBackend(c, vars, bad) + if errRsp == &successResponse { + t.Fatalf("Expected failure, got: %v", errRsp) + } + if errRsp.StatusCode != http.StatusBadRequest { + t.Fatalf("Expected %d. Got: %v", http.StatusBadRequest, errRsp) + } + + cid := "abcdefghi" + jl := endpointJoin{ContainerID: cid} + jlb, err := json.Marshal(jl) + if err != nil { + t.Fatal(err) + } + + _, errRsp = procAttachBackend(c, vars, jlb) + if errRsp != &successResponse { + t.Fatalf("Unexpected failure, got: %v", errRsp) + } + + cli, errRsp := procGetContainers(c, vars, nil) + if errRsp != &successResponse { + t.Fatalf("Unexpected failure, got: %v", errRsp) + } + cl := i2cL(cli) + if len(cl) != 1 { + t.Fatalf("Did not find expected number of containers attached to the service: %d", len(cl)) + } + if cl[0].ID != cid { + t.Fatalf("Did not find expected container attached to the service: %v", cl[0]) + } + + _, errRsp = procUnpublishService(c, vars, nil) + if errRsp.isOK() { + t.Fatalf("Expected failure but succeeded") + } + if errRsp.StatusCode != http.StatusForbidden { + t.Fatalf("Expected %d. Got: %v", http.StatusForbidden, errRsp) + } + + vars[urlEpName] = "endpoint" + _, errRsp = procDetachBackend(c, vars, nil) + if errRsp == &successResponse { + t.Fatalf("Expected failure, got: %v", errRsp) + } + if errRsp.StatusCode != http.StatusNotFound { + t.Fatalf("Expected %d. Got: %v", http.StatusNotFound, errRsp) + } + + vars[urlEpName] = "db" + _, errRsp = procDetachBackend(c, vars, nil) + if errRsp == &successResponse { + t.Fatalf("Expected failure, got: %v", errRsp) + } + if errRsp.StatusCode != http.StatusBadRequest { + t.Fatalf("Expected %d. Got: %v", http.StatusBadRequest, errRsp) + } + + vars[urlCnID] = cid + _, errRsp = procDetachBackend(c, vars, nil) + if errRsp != &successResponse { + t.Fatalf("Unexpected failure, got: %v", errRsp) + } + + cli, errRsp = procGetContainers(c, vars, nil) + if errRsp != &successResponse { + t.Fatalf("Unexpected failure, got: %v", errRsp) + } + cl = i2cL(cli) + if len(cl) != 0 { + t.Fatalf("Did not find expected number of containers attached to the service: %d", len(cl)) + } + + err = ep1.Delete() + if err != nil { + t.Fatal(err) + } +} + +func TestDetectGetNetworksInvalidQueryComposition(t *testing.T) { + c, err := libnetwork.New() + if err != nil { + t.Fatal(err) + } + + vars := map[string]string{urlNwName: "x", urlNwPID: "y"} + _, errRsp := procGetNetworks(c, vars, nil) + if errRsp.StatusCode != http.StatusBadRequest { + t.Fatalf("Expected %d. Got: %v", http.StatusBadRequest, errRsp) + } +} + +func TestDetectGetEndpointsInvalidQueryComposition(t *testing.T) { + defer netutils.SetupTestNetNS(t)() + + c, _ := createTestNetwork(t, "network") + + vars := map[string]string{urlNwName: "network", urlEpName: "x", urlEpPID: "y"} + _, errRsp := procGetEndpoints(c, vars, nil) + if errRsp.StatusCode != http.StatusBadRequest { + t.Fatalf("Expected %d. Got: %v", http.StatusBadRequest, errRsp) + } +} + +func TestDetectGetServicesInvalidQueryComposition(t *testing.T) { + defer netutils.SetupTestNetNS(t)() + + c, _ := createTestNetwork(t, "network") + + vars := map[string]string{urlNwName: "network", urlEpName: "x", urlEpPID: "y"} + _, errRsp := procGetServices(c, vars, nil) + if errRsp.StatusCode != http.StatusBadRequest { + t.Fatalf("Expected %d. Got: %v", http.StatusBadRequest, errRsp) + } +} + +func TestFindNetworkUtilPanic(t *testing.T) { + defer checkPanic(t) + findNetwork(nil, "", -1) +} + +func TestFindNetworkUtil(t *testing.T) { + defer netutils.SetupTestNetNS(t)() + + c, nw := createTestNetwork(t, "network") + nid := nw.ID() + + _, errRsp := findNetwork(c, "", byName) + if errRsp == &successResponse { + t.Fatalf("Expected to fail but succeeded") + } + if errRsp.StatusCode != http.StatusBadRequest { + t.Fatalf("Expected %d, but got: %d", http.StatusBadRequest, errRsp.StatusCode) + } + + n, errRsp := findNetwork(c, nid, byID) + if errRsp != &successResponse { + t.Fatalf("Unexpected failure: %v", errRsp) + } + if n == nil { + t.Fatalf("Unexpected nil libnetwork.Network") + } + if nid != n.ID() { + t.Fatalf("Incorrect libnetwork.Network resource. It has different id: %v", n) + } + if "network" != n.Name() { + t.Fatalf("Incorrect libnetwork.Network resource. It has different name: %v", n) + } + + n, errRsp = findNetwork(c, "network", byName) + if errRsp != &successResponse { + t.Fatalf("Unexpected failure: %v", errRsp) + } + if n == nil { + t.Fatalf("Unexpected nil libnetwork.Network") + } + if nid != n.ID() { + t.Fatalf("Incorrect libnetwork.Network resource. It has different id: %v", n) + } + if "network" != n.Name() { + t.Fatalf("Incorrect libnetwork.Network resource. It has different name: %v", n) + } + + if err := n.Delete(); err != nil { + t.Fatalf("Failed to delete the network: %s", err.Error()) + } + + _, errRsp = findNetwork(c, nid, byID) + if errRsp == &successResponse { + t.Fatalf("Expected to fail but succeeded") + } + if errRsp.StatusCode != http.StatusNotFound { + t.Fatalf("Expected %d, but got: %d", http.StatusNotFound, errRsp.StatusCode) + } + + _, errRsp = findNetwork(c, "network", byName) + if errRsp == &successResponse { + t.Fatalf("Expected to fail but succeeded") + } + if errRsp.StatusCode != http.StatusNotFound { + t.Fatalf("Expected %d, but got: %d", http.StatusNotFound, errRsp.StatusCode) + } +} + +func TestCreateDeleteEndpoints(t *testing.T) { + defer netutils.SetupTestNetNS(t)() + + c, err := libnetwork.New() + if err != nil { + t.Fatal(err) + } + err = c.ConfigureNetworkDriver(bridgeNetType, nil) + if err != nil { + t.Fatal(err) + } + + nc := networkCreate{Name: "firstNet", NetworkType: bridgeNetType} + body, err := json.Marshal(nc) + if err != nil { + t.Fatal(err) + } + + vars := make(map[string]string) + i, errRsp := procCreateNetwork(c, vars, body) + if errRsp != &createdResponse { + t.Fatalf("Unexepected failure: %v", errRsp) + } + nid := i2s(i) + + vbad, err := json.Marshal("bad endppoint create data") + if err != nil { + t.Fatal(err) + } + + vars[urlNwName] = "firstNet" + _, errRsp = procCreateEndpoint(c, vars, vbad) + if errRsp == &createdResponse { + t.Fatalf("Expected to fail but succeeded") + } + + b, err := json.Marshal(endpointCreate{Name: ""}) + if err != nil { + t.Fatal(err) + } + + vars[urlNwName] = "secondNet" + _, errRsp = procCreateEndpoint(c, vars, b) + if errRsp == &createdResponse { + t.Fatalf("Expected to fail but succeeded") + } + + vars[urlNwName] = "firstNet" + _, errRsp = procCreateEndpoint(c, vars, b) + if errRsp == &successResponse { + t.Fatalf("Expected failure but succeeded: %v", errRsp) + } + + b, err = json.Marshal(endpointCreate{Name: "firstEp"}) + if err != nil { + t.Fatal(err) + } + + i, errRsp = procCreateEndpoint(c, vars, b) + if errRsp != &createdResponse { + t.Fatalf("Unexepected failure: %v", errRsp) + } + eid := i2s(i) + + _, errRsp = findEndpoint(c, "myNet", "firstEp", byName, byName) + if errRsp == &successResponse { + t.Fatalf("Expected failure but succeeded: %v", errRsp) + } + + ep0, errRsp := findEndpoint(c, nid, "firstEp", byID, byName) + if errRsp != &successResponse { + t.Fatalf("Unexepected failure: %v", errRsp) + } + + ep1, errRsp := findEndpoint(c, "firstNet", "firstEp", byName, byName) + if errRsp != &successResponse { + t.Fatalf("Unexepected failure: %v", errRsp) + } + + ep2, errRsp := findEndpoint(c, nid, eid, byID, byID) + if errRsp != &successResponse { + t.Fatalf("Unexepected failure: %v", errRsp) + } + + ep3, errRsp := findEndpoint(c, "firstNet", eid, byName, byID) + if errRsp != &successResponse { + t.Fatalf("Unexepected failure: %v", errRsp) + } + + if ep0.ID() != ep1.ID() || ep0.ID() != ep2.ID() || ep0.ID() != ep3.ID() { + t.Fatalf("Diffenrent queries returned different endpoints: \nep0: %v\nep1: %v\nep2: %v\nep3: %v", ep0, ep1, ep2, ep3) + } + + vars = make(map[string]string) + vars[urlNwName] = "" + vars[urlEpName] = "ep1" + _, errRsp = procDeleteEndpoint(c, vars, nil) + if errRsp == &successResponse { + t.Fatalf("Expected failure, got: %v", errRsp) + } + + vars[urlNwName] = "firstNet" + vars[urlEpName] = "" + _, errRsp = procDeleteEndpoint(c, vars, nil) + if errRsp == &successResponse { + t.Fatalf("Expected failure, got: %v", errRsp) + } + + vars[urlEpName] = "ep2" + _, errRsp = procDeleteEndpoint(c, vars, nil) + if errRsp == &successResponse { + t.Fatalf("Expected failure, got: %v", errRsp) + } + + vars[urlEpName] = "firstEp" + _, errRsp = procDeleteEndpoint(c, vars, nil) + if errRsp != &successResponse { + t.Fatalf("Unexepected failure: %v", errRsp) + } + + _, errRsp = findEndpoint(c, "firstNet", "firstEp", byName, byName) + if errRsp == &successResponse { + t.Fatalf("Expected failure, got: %v", errRsp) + } +} + +func TestJoinLeave(t *testing.T) { + defer netutils.SetupTestNetNS(t)() + + c, err := libnetwork.New() + if err != nil { + t.Fatal(err) + } + err = c.ConfigureNetworkDriver(bridgeNetType, nil) + if err != nil { + t.Fatal(err) + } + + nb, err := json.Marshal(networkCreate{Name: "network", NetworkType: bridgeNetType}) + if err != nil { + t.Fatal(err) + } + vars := make(map[string]string) + _, errRsp := procCreateNetwork(c, vars, nb) + if errRsp != &createdResponse { + t.Fatalf("Unexepected failure: %v", errRsp) + } + + eb, err := json.Marshal(endpointCreate{Name: "endpoint"}) + if err != nil { + t.Fatal(err) + } + vars[urlNwName] = "network" + _, errRsp = procCreateEndpoint(c, vars, eb) + if errRsp != &createdResponse { + t.Fatalf("Unexepected failure: %v", errRsp) + } + + vbad, err := json.Marshal("bad data") + if err != nil { + t.Fatal(err) + } + _, errRsp = procJoinEndpoint(c, vars, vbad) + if errRsp == &successResponse { + t.Fatalf("Expected failure, got: %v", errRsp) + } + + vars[urlEpName] = "endpoint" + bad, err := json.Marshal(endpointJoin{}) + if err != nil { + t.Fatal(err) + } + _, errRsp = procJoinEndpoint(c, vars, bad) + if errRsp == &successResponse { + t.Fatalf("Expected failure, got: %v", errRsp) + } + + cid := "abcdefghi" + jl := endpointJoin{ContainerID: cid} + jlb, err := json.Marshal(jl) + if err != nil { + t.Fatal(err) + } + + vars = make(map[string]string) + vars[urlNwName] = "" + vars[urlEpName] = "" + _, errRsp = procJoinEndpoint(c, vars, jlb) + if errRsp == &successResponse { + t.Fatalf("Expected failure, got: %v", errRsp) + } + + vars[urlNwName] = "network" + vars[urlEpName] = "" + _, errRsp = procJoinEndpoint(c, vars, jlb) + if errRsp == &successResponse { + t.Fatalf("Expected failure, got: %v", errRsp) + } + + vars[urlEpName] = "epoint" + _, errRsp = procJoinEndpoint(c, vars, jlb) + if errRsp == &successResponse { + t.Fatalf("Expected failure, got: %v", errRsp) + } + + vars[urlEpName] = "endpoint" + key, errRsp := procJoinEndpoint(c, vars, jlb) + if errRsp != &successResponse { + t.Fatalf("Expected failure, got: %v", errRsp) + } + + keyStr := i2s(key) + if keyStr == "" { + t.Fatalf("Empty sandbox key") + } + _, errRsp = procDeleteEndpoint(c, vars, nil) + if errRsp == &successResponse { + t.Fatalf("Expected failure, got: %v", errRsp) + } + + vars[urlNwName] = "network2" + _, errRsp = procLeaveEndpoint(c, vars, vbad) + if errRsp == &successResponse { + t.Fatalf("Expected failure, got: %v", errRsp) + } + _, errRsp = procLeaveEndpoint(c, vars, bad) + if errRsp == &successResponse { + t.Fatalf("Expected failure, got: %v", errRsp) + } + _, errRsp = procLeaveEndpoint(c, vars, jlb) + if errRsp == &successResponse { + t.Fatalf("Expected failure, got: %v", errRsp) + } + vars = make(map[string]string) + vars[urlNwName] = "" + vars[urlEpName] = "" + _, errRsp = procLeaveEndpoint(c, vars, jlb) + if errRsp == &successResponse { + t.Fatalf("Expected failure, got: %v", errRsp) + } + vars[urlNwName] = "network" + vars[urlEpName] = "" + _, errRsp = procLeaveEndpoint(c, vars, jlb) + if errRsp == &successResponse { + t.Fatalf("Expected failure, got: %v", errRsp) + } + vars[urlEpName] = "2epoint" + _, errRsp = procLeaveEndpoint(c, vars, jlb) + if errRsp == &successResponse { + t.Fatalf("Expected failure, got: %v", errRsp) + } + vars[urlEpName] = "epoint" + vars[urlCnID] = "who" + _, errRsp = procLeaveEndpoint(c, vars, jlb) + if errRsp == &successResponse { + t.Fatalf("Expected failure, got: %v", errRsp) + } + + delete(vars, urlCnID) + vars[urlEpName] = "endpoint" + _, errRsp = procLeaveEndpoint(c, vars, jlb) + if errRsp == &successResponse { + t.Fatalf("Expected failure, got: %v", errRsp) + } + + vars[urlCnID] = cid + _, errRsp = procLeaveEndpoint(c, vars, jlb) + if errRsp != &successResponse { + t.Fatalf("Unexepected failure: %v", errRsp) + } + + _, errRsp = procLeaveEndpoint(c, vars, jlb) + if errRsp == &successResponse { + t.Fatalf("Expected failure, got: %v", errRsp) + } + + _, errRsp = procDeleteEndpoint(c, vars, nil) + if errRsp != &successResponse { + t.Fatalf("Unexepected failure: %v", errRsp) + } +} + +func TestFindEndpointUtilPanic(t *testing.T) { + defer netutils.SetupTestNetNS(t)() + defer checkPanic(t) + c, nw := createTestNetwork(t, "network") + nid := nw.ID() + findEndpoint(c, nid, "", byID, -1) +} + +func TestFindServiceUtilPanic(t *testing.T) { + defer netutils.SetupTestNetNS(t)() + defer checkPanic(t) + c, _ := createTestNetwork(t, "network") + findService(c, "random_service", -1) +} + +func TestFindEndpointUtil(t *testing.T) { + defer netutils.SetupTestNetNS(t)() + + c, nw := createTestNetwork(t, "network") + nid := nw.ID() + + ep, err := nw.CreateEndpoint("secondEp", nil) + if err != nil { + t.Fatal(err) + } + eid := ep.ID() + + _, errRsp := findEndpoint(c, nid, "", byID, byName) + if errRsp == &successResponse { + t.Fatalf("Expected failure, but got: %v", errRsp) + } + if errRsp.StatusCode != http.StatusBadRequest { + t.Fatalf("Expected %d, but got: %d", http.StatusBadRequest, errRsp.StatusCode) + } + + ep0, errRsp := findEndpoint(c, nid, "secondEp", byID, byName) + if errRsp != &successResponse { + t.Fatalf("Unexepected failure: %v", errRsp) + } + + ep1, errRsp := findEndpoint(c, "network", "secondEp", byName, byName) + if errRsp != &successResponse { + t.Fatalf("Unexepected failure: %v", errRsp) + } + + ep2, errRsp := findEndpoint(c, nid, eid, byID, byID) + if errRsp != &successResponse { + t.Fatalf("Unexepected failure: %v", errRsp) + } + + ep3, errRsp := findEndpoint(c, "network", eid, byName, byID) + if errRsp != &successResponse { + t.Fatalf("Unexepected failure: %v", errRsp) + } + + ep4, errRsp := findService(c, "secondEp", byName) + if errRsp != &successResponse { + t.Fatalf("Unexepected failure: %v", errRsp) + } + + ep5, errRsp := findService(c, eid, byID) + if errRsp != &successResponse { + t.Fatalf("Unexepected failure: %v", errRsp) + } + + if ep0 != ep1 || ep0 != ep2 || ep0 != ep3 || ep0 != ep4 || ep0 != ep5 { + t.Fatalf("Diffenrent queries returned different endpoints") + } + + ep.Delete() + + _, errRsp = findEndpoint(c, nid, "secondEp", byID, byName) + if errRsp == &successResponse { + t.Fatalf("Expected failure, but got: %v", errRsp) + } + if errRsp.StatusCode != http.StatusNotFound { + t.Fatalf("Expected %d, but got: %d", http.StatusNotFound, errRsp.StatusCode) + } + + _, errRsp = findEndpoint(c, "network", "secondEp", byName, byName) + if errRsp == &successResponse { + t.Fatalf("Expected failure, but got: %v", errRsp) + } + if errRsp.StatusCode != http.StatusNotFound { + t.Fatalf("Expected %d, but got: %d", http.StatusNotFound, errRsp.StatusCode) + } + + _, errRsp = findEndpoint(c, nid, eid, byID, byID) + if errRsp == &successResponse { + t.Fatalf("Expected failure, but got: %v", errRsp) + } + if errRsp.StatusCode != http.StatusNotFound { + t.Fatalf("Expected %d, but got: %d", http.StatusNotFound, errRsp.StatusCode) + } + + _, errRsp = findEndpoint(c, "network", eid, byName, byID) + if errRsp == &successResponse { + t.Fatalf("Expected failure, but got: %v", errRsp) + } + if errRsp.StatusCode != http.StatusNotFound { + t.Fatalf("Expected %d, but got: %d", http.StatusNotFound, errRsp.StatusCode) + } + + _, errRsp = findService(c, "secondEp", byName) + if errRsp == &successResponse { + t.Fatalf("Expected failure, but got: %v", errRsp) + } + if errRsp.StatusCode != http.StatusNotFound { + t.Fatalf("Expected %d, but got: %d", http.StatusNotFound, errRsp.StatusCode) + } + + _, errRsp = findService(c, eid, byID) + if errRsp == &successResponse { + t.Fatalf("Expected failure, but got: %v", errRsp) + } + if errRsp.StatusCode != http.StatusNotFound { + t.Fatalf("Expected %d, but got: %d", http.StatusNotFound, errRsp.StatusCode) + } +} + +func TestEndpointToService(t *testing.T) { + r := &responseStatus{Status: "this is one endpoint", StatusCode: http.StatusOK} + r = endpointToService(r) + if r.Status != "this is one service" { + t.Fatalf("endpointToService returned unexpected status string: %s", r.Status) + } + + r = &responseStatus{Status: "this is one network", StatusCode: http.StatusOK} + r = endpointToService(r) + if r.Status != "this is one network" { + t.Fatalf("endpointToService returned unexpected status string: %s", r.Status) + } +} + +func checkPanic(t *testing.T) { + if r := recover(); r != nil { + if _, ok := r.(runtime.Error); ok { + panic(r) + } + } else { + t.Fatalf("Expected to panic, but suceeded") + } +} + +func TestDetectNetworkTargetPanic(t *testing.T) { + defer checkPanic(t) + vars := make(map[string]string) + detectNetworkTarget(vars) +} + +func TestDetectEndpointTargetPanic(t *testing.T) { + defer checkPanic(t) + vars := make(map[string]string) + detectEndpointTarget(vars) +} + +func TestResponseStatus(t *testing.T) { + list := []int{ + http.StatusBadGateway, + http.StatusBadRequest, + http.StatusConflict, + http.StatusContinue, + http.StatusExpectationFailed, + http.StatusForbidden, + http.StatusFound, + http.StatusGatewayTimeout, + http.StatusGone, + http.StatusHTTPVersionNotSupported, + http.StatusInternalServerError, + http.StatusLengthRequired, + http.StatusMethodNotAllowed, + http.StatusMovedPermanently, + http.StatusMultipleChoices, + http.StatusNoContent, + http.StatusNonAuthoritativeInfo, + http.StatusNotAcceptable, + http.StatusNotFound, + http.StatusNotModified, + http.StatusPartialContent, + http.StatusPaymentRequired, + http.StatusPreconditionFailed, + http.StatusProxyAuthRequired, + http.StatusRequestEntityTooLarge, + http.StatusRequestTimeout, + http.StatusRequestURITooLong, + http.StatusRequestedRangeNotSatisfiable, + http.StatusResetContent, + http.StatusServiceUnavailable, + http.StatusSwitchingProtocols, + http.StatusTemporaryRedirect, + http.StatusUnauthorized, + http.StatusUnsupportedMediaType, + http.StatusUseProxy, + } + for _, c := range list { + r := responseStatus{StatusCode: c} + if r.isOK() { + t.Fatalf("isOK() returned true for code% d", c) + } + } + + r := responseStatus{StatusCode: http.StatusOK} + if !r.isOK() { + t.Fatalf("isOK() failed") + } + + r = responseStatus{StatusCode: http.StatusCreated} + if !r.isOK() { + t.Fatalf("isOK() failed") + } +} + +// Local structs for end to end testing of api.go +type localReader struct { + data []byte + beBad bool +} + +func newLocalReader(data []byte) *localReader { + lr := &localReader{data: make([]byte, len(data))} + copy(lr.data, data) + return lr +} + +func (l *localReader) Read(p []byte) (n int, err error) { + if l.beBad { + return 0, errors.New("I am a bad reader") + } + if p == nil { + return -1, fmt.Errorf("nil buffer passed") + } + if l.data == nil || len(l.data) == 0 { + return 0, io.EOF + } + copy(p[:], l.data[:]) + return len(l.data), io.EOF +} + +type localResponseWriter struct { + body []byte + statusCode int +} + +func newWriter() *localResponseWriter { + return &localResponseWriter{} +} + +func (f *localResponseWriter) Header() http.Header { + return make(map[string][]string, 0) +} + +func (f *localResponseWriter) Write(data []byte) (int, error) { + if data == nil { + return -1, fmt.Errorf("nil data passed") + } + + f.body = make([]byte, len(data)) + copy(f.body, data) + + return len(f.body), nil +} + +func (f *localResponseWriter) WriteHeader(c int) { + f.statusCode = c +} + +func TestwriteJSON(t *testing.T) { + testCode := 55 + testData, err := json.Marshal("test data") + if err != nil { + t.Fatal(err) + } + + rsp := newWriter() + writeJSON(rsp, testCode, testData) + if rsp.statusCode != testCode { + t.Fatalf("writeJSON() failed to set the status code. Expected %d. Got %d", testCode, rsp.statusCode) + } + if !bytes.Equal(testData, rsp.body) { + t.Fatalf("writeJSON() failed to set the body. Expected %s. Got %s", testData, rsp.body) + } + +} + +func TestHttpHandlerUninit(t *testing.T) { + defer netutils.SetupTestNetNS(t)() + + c, err := libnetwork.New() + if err != nil { + t.Fatal(err) + } + + err = c.ConfigureNetworkDriver(bridgeNetType, nil) + if err != nil { + t.Fatal(err) + } + + h := &httpHandler{c: c} + h.initRouter() + if h.r == nil { + t.Fatalf("initRouter() did not initialize the router") + } + + rsp := newWriter() + req, err := http.NewRequest("GET", "/v1.19/networks", nil) + if err != nil { + t.Fatal(err) + } + + handleRequest := NewHTTPHandler(nil) + handleRequest(rsp, req) + if rsp.statusCode != http.StatusServiceUnavailable { + t.Fatalf("Expected (%d). Got (%d): %s", http.StatusServiceUnavailable, rsp.statusCode, rsp.body) + } + + handleRequest = NewHTTPHandler(c) + + handleRequest(rsp, req) + if rsp.statusCode != http.StatusOK { + t.Fatalf("Expected (%d). Got: (%d): %s", http.StatusOK, rsp.statusCode, rsp.body) + } + + var list []*networkResource + err = json.Unmarshal(rsp.body, &list) + if err != nil { + t.Fatal(err) + } + if len(list) != 0 { + t.Fatalf("Expected empty list. Got %v", list) + } + + n, err := c.NewNetwork(bridgeNetType, "didietro", nil) + if err != nil { + t.Fatal(err) + } + nwr := buildNetworkResource(n) + expected, err := json.Marshal([]*networkResource{nwr}) + if err != nil { + t.Fatal(err) + } + + handleRequest(rsp, req) + if rsp.statusCode != http.StatusOK { + t.Fatalf("Unexpectded failure: (%d): %s", rsp.statusCode, rsp.body) + } + if len(rsp.body) == 0 { + t.Fatalf("Empty list of networks") + } + if bytes.Equal(rsp.body, expected) { + t.Fatalf("Incorrect list of networks in response's body") + } +} + +func TestHttpHandlerBadBody(t *testing.T) { + defer netutils.SetupTestNetNS(t)() + + rsp := newWriter() + + c, err := libnetwork.New() + if err != nil { + t.Fatal(err) + } + handleRequest := NewHTTPHandler(c) + + req, err := http.NewRequest("POST", "/v1.19/networks", &localReader{beBad: true}) + if err != nil { + t.Fatal(err) + } + handleRequest(rsp, req) + if rsp.statusCode != http.StatusBadRequest { + t.Fatalf("Unexpected status code. Expected (%d). Got (%d): %s.", http.StatusBadRequest, rsp.statusCode, string(rsp.body)) + } + + body := []byte{} + lr := newLocalReader(body) + req, err = http.NewRequest("POST", "/v1.19/networks", lr) + if err != nil { + t.Fatal(err) + } + handleRequest(rsp, req) + if rsp.statusCode != http.StatusBadRequest { + t.Fatalf("Unexpected status code. Expected (%d). Got (%d): %s.", http.StatusBadRequest, rsp.statusCode, string(rsp.body)) + } +} + +func TestEndToEnd(t *testing.T) { + defer netutils.SetupTestNetNS(t)() + + rsp := newWriter() + + c, err := libnetwork.New() + if err != nil { + t.Fatal(err) + } + err = c.ConfigureNetworkDriver(bridgeNetType, nil) + if err != nil { + t.Fatal(err) + } + + handleRequest := NewHTTPHandler(c) + + ops := options.Generic{ + netlabel.EnableIPv6: true, + netlabel.GenericData: map[string]string{ + "BridgeName": "cdef", + "FixedCIDRv6": "fe80:2000::1/64", + "EnableIPv6": "true", + "Mtu": "1460", + "EnableIPTables": "true", + "AddressIP": "172.28.30.254/16", + "EnableUserlandProxy": "true", + "AllowNonDefaultBridge": "true", + }, + } + + // Create network + nc := networkCreate{Name: "network-fiftyfive", NetworkType: bridgeNetType, Options: ops} + body, err := json.Marshal(nc) + if err != nil { + t.Fatal(err) + } + lr := newLocalReader(body) + req, err := http.NewRequest("POST", "/v1.19/networks", lr) + if err != nil { + t.Fatal(err) + } + handleRequest(rsp, req) + if rsp.statusCode != http.StatusCreated { + t.Fatalf("Unexpectded status code. Expected (%d). Got (%d): %s.", http.StatusCreated, rsp.statusCode, string(rsp.body)) + } + if len(rsp.body) == 0 { + t.Fatalf("Empty response body") + } + + var nid string + err = json.Unmarshal(rsp.body, &nid) + if err != nil { + t.Fatal(err) + } + + // Query networks collection + req, err = http.NewRequest("GET", "/v1.19/networks?name=", nil) + if err != nil { + t.Fatal(err) + } + handleRequest(rsp, req) + if rsp.statusCode != http.StatusOK { + t.Fatalf("Expected StatusOK. Got (%d): %s", rsp.statusCode, rsp.body) + } + var list []*networkResource + err = json.Unmarshal(rsp.body, &list) + if err != nil { + t.Fatal(err) + } + if len(list) != 0 { + t.Fatalf("Expected empty list. Got %v", list) + } + + req, err = http.NewRequest("GET", "/v1.19/networks", nil) + if err != nil { + t.Fatal(err) + } + handleRequest(rsp, req) + if rsp.statusCode != http.StatusOK { + t.Fatalf("Expected StatusOK. Got (%d): %s", rsp.statusCode, rsp.body) + } + + b0 := make([]byte, len(rsp.body)) + copy(b0, rsp.body) + + req, err = http.NewRequest("GET", "/v1.19/networks?name=network-fiftyfive", nil) + if err != nil { + t.Fatal(err) + } + handleRequest(rsp, req) + if rsp.statusCode != http.StatusOK { + t.Fatalf("Expected StatusOK. Got (%d): %s", rsp.statusCode, rsp.body) + } + + if !bytes.Equal(b0, rsp.body) { + t.Fatalf("Expected same body from GET /networks and GET /networks?name= when only network exist.") + } + + // Query network by name + req, err = http.NewRequest("GET", "/v1.19/networks?name=culo", nil) + if err != nil { + t.Fatal(err) + } + handleRequest(rsp, req) + if rsp.statusCode != http.StatusOK { + t.Fatalf("Expected StatusOK. Got (%d): %s", rsp.statusCode, rsp.body) + } + + err = json.Unmarshal(rsp.body, &list) + if err != nil { + t.Fatal(err) + } + if len(list) != 0 { + t.Fatalf("Expected empty list. Got %v", list) + } + + req, err = http.NewRequest("GET", "/v1.19/networks?name=network-fiftyfive", nil) + if err != nil { + t.Fatal(err) + } + handleRequest(rsp, req) + if rsp.statusCode != http.StatusOK { + t.Fatalf("Unexpectded failure: (%d): %s", rsp.statusCode, rsp.body) + } + + err = json.Unmarshal(rsp.body, &list) + if err != nil { + t.Fatal(err) + } + if len(list) == 0 { + t.Fatalf("Expected non empty list") + } + if list[0].Name != "network-fiftyfive" || nid != list[0].ID { + t.Fatalf("Incongruent resource found: %v", list[0]) + } + + // Query network by partial id + chars := []byte(nid) + partial := string(chars[0 : len(chars)/2]) + req, err = http.NewRequest("GET", "/v1.19/networks?partial-id="+partial, nil) + if err != nil { + t.Fatal(err) + } + handleRequest(rsp, req) + if rsp.statusCode != http.StatusOK { + t.Fatalf("Unexpectded failure: (%d): %s", rsp.statusCode, rsp.body) + } + + err = json.Unmarshal(rsp.body, &list) + if err != nil { + t.Fatal(err) + } + if len(list) == 0 { + t.Fatalf("Expected non empty list") + } + if list[0].Name != "network-fiftyfive" || nid != list[0].ID { + t.Fatalf("Incongruent resource found: %v", list[0]) + } + + // Get network by id + req, err = http.NewRequest("GET", "/v1.19/networks/"+nid, nil) + if err != nil { + t.Fatal(err) + } + handleRequest(rsp, req) + if rsp.statusCode != http.StatusOK { + t.Fatalf("Unexpectded failure: (%d): %s", rsp.statusCode, rsp.body) + } + + var nwr networkResource + err = json.Unmarshal(rsp.body, &nwr) + if err != nil { + t.Fatal(err) + } + if nwr.Name != "network-fiftyfive" || nid != nwr.ID { + t.Fatalf("Incongruent resource found: %v", nwr) + } + + // Create endpoint + eb, err := json.Marshal(endpointCreate{Name: "ep-TwentyTwo"}) + if err != nil { + t.Fatal(err) + } + + lr = newLocalReader(eb) + req, err = http.NewRequest("POST", "/v1.19/networks/"+nid+"/endpoints", lr) + if err != nil { + t.Fatal(err) + } + handleRequest(rsp, req) + if rsp.statusCode != http.StatusCreated { + t.Fatalf("Unexpectded status code. Expected (%d). Got (%d): %s.", http.StatusCreated, rsp.statusCode, string(rsp.body)) + } + if len(rsp.body) == 0 { + t.Fatalf("Empty response body") + } + + var eid string + err = json.Unmarshal(rsp.body, &eid) + if err != nil { + t.Fatal(err) + } + + // Query endpoint(s) + req, err = http.NewRequest("GET", "/v1.19/networks/"+nid+"/endpoints", nil) + if err != nil { + t.Fatal(err) + } + handleRequest(rsp, req) + if rsp.statusCode != http.StatusOK { + t.Fatalf("Expected StatusOK. Got (%d): %s", rsp.statusCode, rsp.body) + } + + req, err = http.NewRequest("GET", "/v1.19/networks/"+nid+"/endpoints?name=bla", nil) + if err != nil { + t.Fatal(err) + } + handleRequest(rsp, req) + if rsp.statusCode != http.StatusOK { + t.Fatalf("Unexpectded failure: (%d): %s", rsp.statusCode, rsp.body) + } + var epList []*endpointResource + err = json.Unmarshal(rsp.body, &epList) + if err != nil { + t.Fatal(err) + } + if len(epList) != 0 { + t.Fatalf("Expected empty list. Got %v", epList) + } + + // Query endpoint by name + req, err = http.NewRequest("GET", "/v1.19/networks/"+nid+"/endpoints?name=ep-TwentyTwo", nil) + if err != nil { + t.Fatal(err) + } + handleRequest(rsp, req) + if rsp.statusCode != http.StatusOK { + t.Fatalf("Unexpectded failure: (%d): %s", rsp.statusCode, rsp.body) + } + + err = json.Unmarshal(rsp.body, &epList) + if err != nil { + t.Fatal(err) + } + if len(epList) == 0 { + t.Fatalf("Empty response body") + } + if epList[0].Name != "ep-TwentyTwo" || eid != epList[0].ID { + t.Fatalf("Incongruent resource found: %v", epList[0]) + } + + // Query endpoint by partial id + chars = []byte(eid) + partial = string(chars[0 : len(chars)/2]) + req, err = http.NewRequest("GET", "/v1.19/networks/"+nid+"/endpoints?partial-id="+partial, nil) + if err != nil { + t.Fatal(err) + } + handleRequest(rsp, req) + if rsp.statusCode != http.StatusOK { + t.Fatalf("Unexpectded failure: (%d): %s", rsp.statusCode, rsp.body) + } + + err = json.Unmarshal(rsp.body, &epList) + if err != nil { + t.Fatal(err) + } + if len(epList) == 0 { + t.Fatalf("Empty response body") + } + if epList[0].Name != "ep-TwentyTwo" || eid != epList[0].ID { + t.Fatalf("Incongruent resource found: %v", epList[0]) + } + + // Get endpoint by id + req, err = http.NewRequest("GET", "/v1.19/networks/"+nid+"/endpoints/"+eid, nil) + if err != nil { + t.Fatal(err) + } + handleRequest(rsp, req) + if rsp.statusCode != http.StatusOK { + t.Fatalf("Unexpectded failure: (%d): %s", rsp.statusCode, rsp.body) + } + + var epr endpointResource + err = json.Unmarshal(rsp.body, &epr) + if err != nil { + t.Fatal(err) + } + if epr.Name != "ep-TwentyTwo" || epr.ID != eid { + t.Fatalf("Incongruent resource found: %v", epr) + } +} + +type bre struct{} + +func (b *bre) Error() string { + return "I am a bad request error" +} +func (b *bre) BadRequest() {} + +type nfe struct{} + +func (n *nfe) Error() string { + return "I am a not found error" +} +func (n *nfe) NotFound() {} + +type forb struct{} + +func (f *forb) Error() string { + return "I am a bad request error" +} +func (f *forb) Forbidden() {} + +type notimpl struct{} + +func (nip *notimpl) Error() string { + return "I am a not implemented error" +} +func (nip *notimpl) NotImplemented() {} + +type inter struct{} + +func (it *inter) Error() string { + return "I am a internal error" +} +func (it *inter) Internal() {} + +type tout struct{} + +func (to *tout) Error() string { + return "I am a timeout error" +} +func (to *tout) Timeout() {} + +type noserv struct{} + +func (nos *noserv) Error() string { + return "I am a no service error" +} +func (nos *noserv) NoService() {} + +type notclassified struct{} + +func (noc *notclassified) Error() string { + return "I am a non classified error" +} + +func TestErrorConversion(t *testing.T) { + if convertNetworkError(new(bre)).StatusCode != http.StatusBadRequest { + t.Fatalf("Failed to recognize BadRequest error") + } + + if convertNetworkError(new(nfe)).StatusCode != http.StatusNotFound { + t.Fatalf("Failed to recognize NotFound error") + } + + if convertNetworkError(new(forb)).StatusCode != http.StatusForbidden { + t.Fatalf("Failed to recognize Forbidden error") + } + + if convertNetworkError(new(notimpl)).StatusCode != http.StatusNotImplemented { + t.Fatalf("Failed to recognize NotImplemented error") + } + + if convertNetworkError(new(inter)).StatusCode != http.StatusInternalServerError { + t.Fatalf("Failed to recognize Internal error") + } + + if convertNetworkError(new(tout)).StatusCode != http.StatusRequestTimeout { + t.Fatalf("Failed to recognize Timeout error") + } + + if convertNetworkError(new(noserv)).StatusCode != http.StatusServiceUnavailable { + t.Fatalf("Failed to recognize No Service error") + } + + if convertNetworkError(new(notclassified)).StatusCode != http.StatusInternalServerError { + t.Fatalf("Failed to recognize not classified error as Internal error") + } +} + +func TestFieldRegex(t *testing.T) { + pr := regexp.MustCompile(regex) + qr := regexp.MustCompile(`^` + qregx + `$`) // mux compiles it like this + + if pr.MatchString("") { + t.Fatalf("Unexpected match") + } + if !qr.MatchString("") { + t.Fatalf("Unexpected match failure") + } + + if pr.MatchString(":") { + t.Fatalf("Unexpected match") + } + if qr.MatchString(":") { + t.Fatalf("Unexpected match") + } + + if pr.MatchString(".") { + t.Fatalf("Unexpected match") + } + if qr.MatchString(".") { + t.Fatalf("Unexpected match") + } +} diff --git a/Godeps/_workspace/src/github.com/docker/libnetwork/api/types.go b/Godeps/_workspace/src/github.com/docker/libnetwork/api/types.go new file mode 100644 index 000000000..72f20db24 --- /dev/null +++ b/Godeps/_workspace/src/github.com/docker/libnetwork/api/types.go @@ -0,0 +1,81 @@ +package api + +import "github.com/docker/libnetwork/types" + +/*********** + Resources +************/ + +// networkResource is the body of the "get network" http response message +type networkResource struct { + Name string `json:"name"` + ID string `json:"id"` + Type string `json:"type"` + Endpoints []*endpointResource `json:"endpoints"` +} + +// endpointResource is the body of the "get endpoint" http response message +type endpointResource struct { + Name string `json:"name"` + ID string `json:"id"` + Network string `json:"network"` +} + +// containerResource is the body of "get service backend" response message +type containerResource struct { + ID string `json:"id"` + // will add more fields once labels change is in +} + +/*********** + Body types + ************/ + +// networkCreate is the expected body of the "create network" http request message +type networkCreate struct { + Name string `json:"name"` + NetworkType string `json:"network_type"` + Options map[string]interface{} `json:"options"` +} + +// endpointCreate represents the body of the "create endpoint" http request message +type endpointCreate struct { + Name string `json:"name"` + ExposedPorts []types.TransportPort `json:"exposed_ports"` + PortMapping []types.PortBinding `json:"port_mapping"` +} + +// endpointJoin represents the expected body of the "join endpoint" or "leave endpoint" http request messages +type endpointJoin struct { + ContainerID string `json:"container_id"` + HostName string `json:"host_name"` + DomainName string `json:"domain_name"` + HostsPath string `json:"hosts_path"` + ResolvConfPath string `json:"resolv_conf_path"` + DNS []string `json:"dns"` + ExtraHosts []endpointExtraHost `json:"extra_hosts"` + ParentUpdates []endpointParentUpdate `json:"parent_updates"` + UseDefaultSandbox bool `json:"use_default_sandbox"` +} + +// servicePublish represents the body of the "publish service" http request message +type servicePublish struct { + Name string `json:"name"` + Network string `json:"network_name"` + ExposedPorts []types.TransportPort `json:"exposed_ports"` + PortMapping []types.PortBinding `json:"port_mapping"` +} + +// EndpointExtraHost represents the extra host object +type endpointExtraHost struct { + Name string `json:"name"` + Address string `json:"address"` +} + +// EndpointParentUpdate is the object carrying the information about the +// endpoint parent that needs to be updated +type endpointParentUpdate struct { + EndpointID string `json:"endpoint_id"` + Name string `json:"name"` + Address string `json:"address"` +} diff --git a/Godeps/_workspace/src/github.com/docker/libnetwork/bitseq/sequence.go b/Godeps/_workspace/src/github.com/docker/libnetwork/bitseq/sequence.go new file mode 100644 index 000000000..d71d76b3e --- /dev/null +++ b/Godeps/_workspace/src/github.com/docker/libnetwork/bitseq/sequence.go @@ -0,0 +1,541 @@ +// Package bitseq provides a structure and utilities for representing long bitmask +// as sequence of run-lenght encoded blocks. It operates direclty on the encoded +// representation, it does not decode/encode. +package bitseq + +import ( + "encoding/binary" + "fmt" + "sync" + + "github.com/docker/libnetwork/datastore" + "github.com/docker/libnetwork/types" +) + +// block sequence constants +// If needed we can think of making these configurable +const ( + blockLen = uint32(32) + blockBytes = blockLen / 8 + blockMAX = uint32(1<%s", s.block, s.count, nextBlock) +} + +// GetAvailableBit returns the position of the first unset bit in the bitmask represented by this sequence +func (s *sequence) getAvailableBit() (uint32, uint32, error) { + if s.block == blockMAX || s.count == 0 { + return invalidPos, invalidPos, fmt.Errorf("no available bit") + } + bits := uint32(0) + bitSel := blockFirstBit + for bitSel > 0 && s.block&bitSel != 0 { + bitSel >>= 1 + bits++ + } + return bits / 8, bits % 8, nil +} + +// GetCopy returns a copy of the linked list rooted at this node +func (s *sequence) getCopy() *sequence { + n := &sequence{block: s.block, count: s.count} + pn := n + ps := s.next + for ps != nil { + pn.next = &sequence{block: ps.block, count: ps.count} + pn = pn.next + ps = ps.next + } + return n +} + +// Equal checks if this sequence is equal to the passed one +func (s *sequence) equal(o *sequence) bool { + this := s + other := o + for this != nil { + if other == nil { + return false + } + if this.block != other.block || this.count != other.count { + return false + } + this = this.next + other = other.next + } + // Check if other is longer than this + if other != nil { + return false + } + return true +} + +// ToByteArray converts the sequence into a byte array +func (s *sequence) toByteArray() ([]byte, error) { + var bb []byte + + p := s + for p != nil { + b := make([]byte, 8) + binary.BigEndian.PutUint32(b[0:], p.block) + binary.BigEndian.PutUint32(b[4:], p.count) + bb = append(bb, b...) + p = p.next + } + + return bb, nil +} + +// fromByteArray construct the sequence from the byte array +func (s *sequence) fromByteArray(data []byte) error { + l := len(data) + if l%8 != 0 { + return fmt.Errorf("cannot deserialize byte sequence of lenght %d (%v)", l, data) + } + + p := s + i := 0 + for { + p.block = binary.BigEndian.Uint32(data[i : i+4]) + p.count = binary.BigEndian.Uint32(data[i+4 : i+8]) + i += 8 + if i == l { + break + } + p.next = &sequence{} + p = p.next + } + + return nil +} + +func (h *Handle) getCopy() *Handle { + return &Handle{ + bits: h.bits, + unselected: h.unselected, + head: h.head.getCopy(), + app: h.app, + id: h.id, + dbIndex: h.dbIndex, + dbExists: h.dbExists, + store: h.store, + } +} + +// SetAny atomically sets the first unset bit in the sequence and returns the corresponding ordinal +func (h *Handle) SetAny() (uint32, error) { + if h.Unselected() == 0 { + return invalidPos, errNoBitAvailable + } + return h.set(0, true, false) +} + +// Set atomically sets the corresponding bit in the sequence +func (h *Handle) Set(ordinal uint32) error { + if err := h.validateOrdinal(ordinal); err != nil { + return err + } + _, err := h.set(ordinal, false, false) + return err +} + +// Unset atomically unsets the corresponding bit in the sequence +func (h *Handle) Unset(ordinal uint32) error { + if err := h.validateOrdinal(ordinal); err != nil { + return err + } + _, err := h.set(ordinal, false, true) + return err +} + +// IsSet atomically checks if the ordinal bit is set. In case ordinal +// is outside of the bit sequence limits, false is returned. +func (h *Handle) IsSet(ordinal uint32) bool { + if err := h.validateOrdinal(ordinal); err != nil { + return false + } + h.Lock() + _, _, err := checkIfAvailable(h.head, ordinal) + h.Unlock() + return err != nil +} + +// set/reset the bit +func (h *Handle) set(ordinal uint32, any bool, release bool) (uint32, error) { + var ( + bitPos uint32 + bytePos uint32 + ret uint32 + err error + ) + + for { + h.Lock() + // Get position if available + if release { + bytePos, bitPos = ordinalToPos(ordinal) + } else { + if any { + bytePos, bitPos, err = getFirstAvailable(h.head) + ret = posToOrdinal(bytePos, bitPos) + } else { + bytePos, bitPos, err = checkIfAvailable(h.head, ordinal) + ret = ordinal + } + } + if err != nil { + h.Unlock() + return ret, err + } + + // Create a private copy of h and work on it, also copy the current db index + nh := h.getCopy() + ci := h.dbIndex + h.Unlock() + + nh.head = pushReservation(bytePos, bitPos, nh.head, release) + if release { + nh.unselected++ + } else { + nh.unselected-- + } + + // Attempt to write private copy to store + if err := nh.writeToStore(); err != nil { + if _, ok := err.(types.RetryError); !ok { + return ret, fmt.Errorf("internal failure while setting the bit: %v", err) + } + // Retry + continue + } + + // Unless unexpected error, save private copy to local copy + h.Lock() + defer h.Unlock() + if h.dbIndex != ci { + return ret, fmt.Errorf("unexected database index change") + } + h.unselected = nh.unselected + h.head = nh.head + h.dbExists = nh.dbExists + h.dbIndex = nh.dbIndex + return ret, nil + } +} + +// checks is needed because to cover the case where the number of bits is not a multiple of blockLen +func (h *Handle) validateOrdinal(ordinal uint32) error { + if ordinal > h.bits { + return fmt.Errorf("bit does not belong to the sequence") + } + return nil +} + +// Destroy removes from the datastore the data belonging to this handle +func (h *Handle) Destroy() { + h.deleteFromStore() +} + +// ToByteArray converts this handle's data into a byte array +func (h *Handle) ToByteArray() ([]byte, error) { + + h.Lock() + defer h.Unlock() + ba := make([]byte, 8) + binary.BigEndian.PutUint32(ba[0:], h.bits) + binary.BigEndian.PutUint32(ba[4:], h.unselected) + bm, err := h.head.toByteArray() + if err != nil { + return nil, fmt.Errorf("failed to serialize head: %s", err.Error()) + } + ba = append(ba, bm...) + + return ba, nil +} + +// FromByteArray reads his handle's data from a byte array +func (h *Handle) FromByteArray(ba []byte) error { + if ba == nil { + return fmt.Errorf("nil byte array") + } + + nh := &sequence{} + err := nh.fromByteArray(ba[8:]) + if err != nil { + return fmt.Errorf("failed to deserialize head: %s", err.Error()) + } + + h.Lock() + h.head = nh + h.bits = binary.BigEndian.Uint32(ba[0:4]) + h.unselected = binary.BigEndian.Uint32(ba[4:8]) + h.Unlock() + + return nil +} + +// Bits returns the length of the bit sequence +func (h *Handle) Bits() uint32 { + return h.bits +} + +// Unselected returns the number of bits which are not selected +func (h *Handle) Unselected() uint32 { + h.Lock() + defer h.Unlock() + return h.unselected +} + +func (h *Handle) String() string { + h.Lock() + defer h.Unlock() + return fmt.Sprintf("App: %s, ID: %s, DBIndex: 0x%x, bits: %d, unselected: %d, sequence: %s", + h.app, h.id, h.dbIndex, h.bits, h.unselected, h.head.toString()) +} + +// getFirstAvailable looks for the first unset bit in passed mask +func getFirstAvailable(head *sequence) (uint32, uint32, error) { + byteIndex := uint32(0) + current := head + for current != nil { + if current.block != blockMAX { + bytePos, bitPos, err := current.getAvailableBit() + return byteIndex + bytePos, bitPos, err + } + byteIndex += current.count * blockBytes + current = current.next + } + return invalidPos, invalidPos, errNoBitAvailable +} + +// checkIfAvailable checks if the bit correspondent to the specified ordinal is unset +// If the ordinal is beyond the sequence limits, a negative response is returned +func checkIfAvailable(head *sequence, ordinal uint32) (uint32, uint32, error) { + bytePos := ordinal / 8 + bitPos := ordinal % 8 + + // Find the sequence containing this byte + current, _, _, inBlockBytePos := findSequence(head, bytePos) + if current != nil { + // Check whether the bit corresponding to the ordinal address is unset + bitSel := blockFirstBit >> (inBlockBytePos*8 + bitPos) + if current.block&bitSel == 0 { + return bytePos, bitPos, nil + } + } + + return invalidPos, invalidPos, fmt.Errorf("requested bit is not available") +} + +// Given the byte position and the sequences list head, return the pointer to the +// sequence containing the byte (current), the pointer to the previous sequence, +// the number of blocks preceding the block containing the byte inside the current sequence. +// If bytePos is outside of the list, function will return (nil, nil, 0, invalidPos) +func findSequence(head *sequence, bytePos uint32) (*sequence, *sequence, uint32, uint32) { + // Find the sequence containing this byte + previous := head + current := head + n := bytePos + for current.next != nil && n >= (current.count*blockBytes) { // Nil check for less than 32 addresses masks + n -= (current.count * blockBytes) + previous = current + current = current.next + } + + // If byte is outside of the list, let caller know + if n >= (current.count * blockBytes) { + return nil, nil, 0, invalidPos + } + + // Find the byte position inside the block and the number of blocks + // preceding the block containing the byte inside this sequence + precBlocks := n / blockBytes + inBlockBytePos := bytePos % blockBytes + + return current, previous, precBlocks, inBlockBytePos +} + +// PushReservation pushes the bit reservation inside the bitmask. +// Given byte and bit positions, identify the sequence (current) which holds the block containing the affected bit. +// Create a new block with the modified bit according to the operation (allocate/release). +// Create a new sequence containing the new block and insert it in the proper position. +// Remove current sequence if empty. +// Check if new sequence can be merged with neighbour (previous/next) sequences. +// +// +// Identify "current" sequence containing block: +// [prev seq] [current seq] [next seq] +// +// Based on block position, resulting list of sequences can be any of three forms: +// +// block position Resulting list of sequences +// A) block is first in current: [prev seq] [new] [modified current seq] [next seq] +// B) block is last in current: [prev seq] [modified current seq] [new] [next seq] +// C) block is in the middle of current: [prev seq] [curr pre] [new] [curr post] [next seq] +func pushReservation(bytePos, bitPos uint32, head *sequence, release bool) *sequence { + // Store list's head + newHead := head + + // Find the sequence containing this byte + current, previous, precBlocks, inBlockBytePos := findSequence(head, bytePos) + if current == nil { + return newHead + } + + // Construct updated block + bitSel := blockFirstBit >> (inBlockBytePos*8 + bitPos) + newBlock := current.block + if release { + newBlock &^= bitSel + } else { + newBlock |= bitSel + } + + // Quit if it was a redundant request + if current.block == newBlock { + return newHead + } + + // Current sequence inevitably looses one block, upadate count + current.count-- + + // Create new sequence + newSequence := &sequence{block: newBlock, count: 1} + + // Insert the new sequence in the list based on block position + if precBlocks == 0 { // First in sequence (A) + newSequence.next = current + if current == head { + newHead = newSequence + previous = newHead + } else { + previous.next = newSequence + } + removeCurrentIfEmpty(&newHead, newSequence, current) + mergeSequences(previous) + } else if precBlocks == current.count-2 { // Last in sequence (B) + newSequence.next = current.next + current.next = newSequence + mergeSequences(current) + } else { // In between the sequence (C) + currPre := &sequence{block: current.block, count: precBlocks, next: newSequence} + currPost := current + currPost.count -= precBlocks + newSequence.next = currPost + if currPost == head { + newHead = currPre + } else { + previous.next = currPre + } + // No merging or empty current possible here + } + + return newHead +} + +// Removes the current sequence from the list if empty, adjusting the head pointer if needed +func removeCurrentIfEmpty(head **sequence, previous, current *sequence) { + if current.count == 0 { + if current == *head { + *head = current.next + } else { + previous.next = current.next + current = current.next + } + } +} + +// Given a pointer to a sequence, it checks if it can be merged with any following sequences +// It stops when no more merging is possible. +// TODO: Optimization: only attempt merge from start to end sequence, no need to scan till the end of the list +func mergeSequences(seq *sequence) { + if seq != nil { + // Merge all what possible from seq + for seq.next != nil && seq.block == seq.next.block { + seq.count += seq.next.count + seq.next = seq.next.next + } + // Move to next + mergeSequences(seq.next) + } +} + +func getNumBlocks(numBits uint32) uint32 { + numBlocks := numBits / blockLen + if numBits%blockLen != 0 { + numBlocks++ + } + return numBlocks +} + +func ordinalToPos(ordinal uint32) (uint32, uint32) { + return ordinal / 8, ordinal % 8 +} + +func posToOrdinal(bytePos, bitPos uint32) uint32 { + return bytePos*8 + bitPos +} diff --git a/Godeps/_workspace/src/github.com/docker/libnetwork/bitseq/sequence_test.go b/Godeps/_workspace/src/github.com/docker/libnetwork/bitseq/sequence_test.go new file mode 100644 index 000000000..26d764334 --- /dev/null +++ b/Godeps/_workspace/src/github.com/docker/libnetwork/bitseq/sequence_test.go @@ -0,0 +1,497 @@ +package bitseq + +import ( + "testing" + + _ "github.com/docker/libnetwork/netutils" +) + +func TestSequenceGetAvailableBit(t *testing.T) { + input := []struct { + head *sequence + bytePos uint32 + bitPos uint32 + }{ + {&sequence{block: 0x0, count: 0}, invalidPos, invalidPos}, + {&sequence{block: 0x0, count: 1}, 0, 0}, + {&sequence{block: 0x0, count: 100}, 0, 0}, + + {&sequence{block: 0x80000000, count: 0}, invalidPos, invalidPos}, + {&sequence{block: 0x80000000, count: 1}, 0, 1}, + {&sequence{block: 0x80000000, count: 100}, 0, 1}, + + {&sequence{block: 0xFF000000, count: 0}, invalidPos, invalidPos}, + {&sequence{block: 0xFF000000, count: 1}, 1, 0}, + {&sequence{block: 0xFF000000, count: 100}, 1, 0}, + + {&sequence{block: 0xFF800000, count: 0}, invalidPos, invalidPos}, + {&sequence{block: 0xFF800000, count: 1}, 1, 1}, + {&sequence{block: 0xFF800000, count: 100}, 1, 1}, + + {&sequence{block: 0xFFC0FF00, count: 0}, invalidPos, invalidPos}, + {&sequence{block: 0xFFC0FF00, count: 1}, 1, 2}, + {&sequence{block: 0xFFC0FF00, count: 100}, 1, 2}, + + {&sequence{block: 0xFFE0FF00, count: 0}, invalidPos, invalidPos}, + {&sequence{block: 0xFFE0FF00, count: 1}, 1, 3}, + {&sequence{block: 0xFFE0FF00, count: 100}, 1, 3}, + + {&sequence{block: 0xFFFEFF00, count: 0}, invalidPos, invalidPos}, + {&sequence{block: 0xFFFEFF00, count: 1}, 1, 7}, + {&sequence{block: 0xFFFEFF00, count: 100}, 1, 7}, + + {&sequence{block: 0xFFFFC0FF, count: 0}, invalidPos, invalidPos}, + {&sequence{block: 0xFFFFC0FF, count: 1}, 2, 2}, + {&sequence{block: 0xFFFFC0FF, count: 100}, 2, 2}, + + {&sequence{block: 0xFFFFFF00, count: 0}, invalidPos, invalidPos}, + {&sequence{block: 0xFFFFFF00, count: 1}, 3, 0}, + {&sequence{block: 0xFFFFFF00, count: 100}, 3, 0}, + + {&sequence{block: 0xFFFFFFFE, count: 0}, invalidPos, invalidPos}, + {&sequence{block: 0xFFFFFFFE, count: 1}, 3, 7}, + {&sequence{block: 0xFFFFFFFE, count: 100}, 3, 7}, + + {&sequence{block: 0xFFFFFFFF, count: 0}, invalidPos, invalidPos}, + {&sequence{block: 0xFFFFFFFF, count: 1}, invalidPos, invalidPos}, + {&sequence{block: 0xFFFFFFFF, count: 100}, invalidPos, invalidPos}, + } + + for n, i := range input { + b, bb, err := i.head.getAvailableBit() + if b != i.bytePos || bb != i.bitPos { + t.Fatalf("Error in sequence.getAvailableBit() (%d).\nExp: (%d, %d)\nGot: (%d, %d), err: %v", n, i.bytePos, i.bitPos, b, bb, err) + } + } +} + +func TestSequenceEqual(t *testing.T) { + input := []struct { + first *sequence + second *sequence + areEqual bool + }{ + {&sequence{block: 0x0, count: 8, next: nil}, &sequence{block: 0x0, count: 8}, true}, + {&sequence{block: 0x0, count: 0, next: nil}, &sequence{block: 0x0, count: 0}, true}, + {&sequence{block: 0x0, count: 2, next: nil}, &sequence{block: 0x0, count: 1, next: &sequence{block: 0x0, count: 1}}, false}, + {&sequence{block: 0x0, count: 2, next: &sequence{block: 0x1, count: 1}}, &sequence{block: 0x0, count: 2}, false}, + + {&sequence{block: 0x12345678, count: 8, next: nil}, &sequence{block: 0x12345678, count: 8}, true}, + {&sequence{block: 0x12345678, count: 8, next: nil}, &sequence{block: 0x12345678, count: 9}, false}, + {&sequence{block: 0x12345678, count: 1, next: &sequence{block: 0XFFFFFFFF, count: 1}}, &sequence{block: 0x12345678, count: 1}, false}, + {&sequence{block: 0x12345678, count: 1}, &sequence{block: 0x12345678, count: 1, next: &sequence{block: 0XFFFFFFFF, count: 1}}, false}, + } + + for n, i := range input { + if i.areEqual != i.first.equal(i.second) { + t.Fatalf("Error in sequence.equal() (%d).\nExp: %t\nGot: %t,", n, i.areEqual, !i.areEqual) + } + } +} + +func TestSequenceCopy(t *testing.T) { + s := getTestSequence() + n := s.getCopy() + if !s.equal(n) { + t.Fatalf("copy of s failed") + } + if n == s { + t.Fatalf("not true copy of s") + } +} + +func TestGetFirstAvailable(t *testing.T) { + input := []struct { + mask *sequence + bytePos uint32 + bitPos uint32 + }{ + {&sequence{block: 0xffffffff, count: 2048}, invalidPos, invalidPos}, + {&sequence{block: 0x0, count: 8}, 0, 0}, + {&sequence{block: 0x80000000, count: 8}, 0, 1}, + {&sequence{block: 0xC0000000, count: 8}, 0, 2}, + {&sequence{block: 0xE0000000, count: 8}, 0, 3}, + {&sequence{block: 0xF0000000, count: 8}, 0, 4}, + {&sequence{block: 0xF8000000, count: 8}, 0, 5}, + {&sequence{block: 0xFC000000, count: 8}, 0, 6}, + {&sequence{block: 0xFE000000, count: 8}, 0, 7}, + + {&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0x00000000, count: 1, next: &sequence{block: 0xffffffff, count: 6}}}, 4, 0}, + {&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0x80000000, count: 1, next: &sequence{block: 0xffffffff, count: 6}}}, 4, 1}, + {&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0xC0000000, count: 1, next: &sequence{block: 0xffffffff, count: 6}}}, 4, 2}, + {&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0xE0000000, count: 1, next: &sequence{block: 0xffffffff, count: 6}}}, 4, 3}, + {&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0xF0000000, count: 1, next: &sequence{block: 0xffffffff, count: 6}}}, 4, 4}, + {&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0xF8000000, count: 1, next: &sequence{block: 0xffffffff, count: 6}}}, 4, 5}, + {&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0xFC000000, count: 1, next: &sequence{block: 0xffffffff, count: 6}}}, 4, 6}, + {&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0xFE000000, count: 1, next: &sequence{block: 0xffffffff, count: 6}}}, 4, 7}, + + {&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0xFF000000, count: 1, next: &sequence{block: 0xffffffff, count: 6}}}, 5, 0}, + {&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0xFF800000, count: 1, next: &sequence{block: 0xffffffff, count: 6}}}, 5, 1}, + {&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0xFFC00000, count: 1, next: &sequence{block: 0xffffffff, count: 6}}}, 5, 2}, + {&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0xFFE00000, count: 1, next: &sequence{block: 0xffffffff, count: 6}}}, 5, 3}, + {&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0xFFF00000, count: 1, next: &sequence{block: 0xffffffff, count: 6}}}, 5, 4}, + {&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0xFFF80000, count: 1, next: &sequence{block: 0xffffffff, count: 6}}}, 5, 5}, + {&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0xFFFC0000, count: 1, next: &sequence{block: 0xffffffff, count: 6}}}, 5, 6}, + {&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0xFFFE0000, count: 1, next: &sequence{block: 0xffffffff, count: 6}}}, 5, 7}, + + {&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0xfffffffe, count: 1, next: &sequence{block: 0xffffffff, count: 6}}}, 7, 7}, + + {&sequence{block: 0xffffffff, count: 2, next: &sequence{block: 0x0, count: 6}}, 8, 0}, + } + + for n, i := range input { + bytePos, bitPos, _ := getFirstAvailable(i.mask) + if bytePos != i.bytePos || bitPos != i.bitPos { + t.Fatalf("Error in (%d) getFirstAvailable(). Expected (%d, %d). Got (%d, %d)", n, i.bytePos, i.bitPos, bytePos, bitPos) + } + } +} + +func TestFindSequence(t *testing.T) { + input := []struct { + head *sequence + bytePos uint32 + precBlocks uint32 + inBlockBytePos uint32 + }{ + {&sequence{block: 0xffffffff, count: 0}, 0, 0, invalidPos}, + {&sequence{block: 0xffffffff, count: 0}, 31, 0, invalidPos}, + {&sequence{block: 0xffffffff, count: 0}, 100, 0, invalidPos}, + + {&sequence{block: 0x0, count: 1}, 0, 0, 0}, + {&sequence{block: 0x0, count: 1}, 1, 0, 1}, + {&sequence{block: 0x0, count: 1}, 31, 0, invalidPos}, + {&sequence{block: 0x0, count: 1}, 60, 0, invalidPos}, + + {&sequence{block: 0xffffffff, count: 10}, 0, 0, 0}, + {&sequence{block: 0xffffffff, count: 10}, 3, 0, 3}, + {&sequence{block: 0xffffffff, count: 10}, 4, 1, 0}, + {&sequence{block: 0xffffffff, count: 10}, 7, 1, 3}, + {&sequence{block: 0xffffffff, count: 10}, 8, 2, 0}, + {&sequence{block: 0xffffffff, count: 10}, 39, 9, 3}, + + {&sequence{block: 0xffffffff, count: 10, next: &sequence{block: 0xcc000000, count: 10}}, 79, 9, 3}, + {&sequence{block: 0xffffffff, count: 10, next: &sequence{block: 0xcc000000, count: 10}}, 80, 0, invalidPos}, + } + + for n, i := range input { + _, _, precBlocks, inBlockBytePos := findSequence(i.head, i.bytePos) + if precBlocks != i.precBlocks || inBlockBytePos != i.inBlockBytePos { + t.Fatalf("Error in (%d) findSequence(). Expected (%d, %d). Got (%d, %d)", n, i.precBlocks, i.inBlockBytePos, precBlocks, inBlockBytePos) + } + } +} + +func TestCheckIfAvailable(t *testing.T) { + input := []struct { + head *sequence + ordinal uint32 + bytePos uint32 + bitPos uint32 + }{ + {&sequence{block: 0xffffffff, count: 0}, 0, invalidPos, invalidPos}, + {&sequence{block: 0xffffffff, count: 0}, 31, invalidPos, invalidPos}, + {&sequence{block: 0xffffffff, count: 0}, 100, invalidPos, invalidPos}, + + {&sequence{block: 0x0, count: 1}, 0, 0, 0}, + {&sequence{block: 0x0, count: 1}, 1, 0, 1}, + {&sequence{block: 0x0, count: 1}, 31, 3, 7}, + {&sequence{block: 0x0, count: 1}, 60, invalidPos, invalidPos}, + + {&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0x800000ff, count: 1}}, 31, invalidPos, invalidPos}, + {&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0x800000ff, count: 1}}, 32, invalidPos, invalidPos}, + {&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0x800000ff, count: 1}}, 33, 4, 1}, + {&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0xC00000ff, count: 1}}, 33, invalidPos, invalidPos}, + {&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0xC00000ff, count: 1}}, 34, 4, 2}, + + {&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0xC00000ff, count: 1, next: &sequence{block: 0x0, count: 1}}}, 55, 6, 7}, + {&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0xC00000ff, count: 1, next: &sequence{block: 0x0, count: 1}}}, 56, invalidPos, invalidPos}, + {&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0xC00000ff, count: 1, next: &sequence{block: 0x0, count: 1}}}, 63, invalidPos, invalidPos}, + + {&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0xC00000ff, count: 1, next: &sequence{block: 0x0, count: 1}}}, 64, 8, 0}, + {&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0xC00000ff, count: 1, next: &sequence{block: 0x0, count: 1}}}, 95, 11, 7}, + {&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0xC00000ff, count: 1, next: &sequence{block: 0x0, count: 1}}}, 96, invalidPos, invalidPos}, + } + + for n, i := range input { + bytePos, bitPos, err := checkIfAvailable(i.head, i.ordinal) + if bytePos != i.bytePos || bitPos != i.bitPos { + t.Fatalf("Error in (%d) checkIfAvailable(ord:%d). Expected (%d, %d). Got (%d, %d). err: %v", n, i.ordinal, i.bytePos, i.bitPos, bytePos, bitPos, err) + } + } +} + +func TestMergeSequences(t *testing.T) { + input := []struct { + original *sequence + merged *sequence + }{ + {&sequence{block: 0xFE000000, count: 8, next: &sequence{block: 0xFE000000, count: 2}}, &sequence{block: 0xFE000000, count: 10}}, + {&sequence{block: 0xFFFFFFFF, count: 8, next: &sequence{block: 0xFFFFFFFF, count: 1}}, &sequence{block: 0xFFFFFFFF, count: 9}}, + {&sequence{block: 0xFFFFFFFF, count: 1, next: &sequence{block: 0xFFFFFFFF, count: 8}}, &sequence{block: 0xFFFFFFFF, count: 9}}, + + {&sequence{block: 0xFFFFFFF0, count: 8, next: &sequence{block: 0xFFFFFFF0, count: 1}}, &sequence{block: 0xFFFFFFF0, count: 9}}, + {&sequence{block: 0xFFFFFFF0, count: 1, next: &sequence{block: 0xFFFFFFF0, count: 8}}, &sequence{block: 0xFFFFFFF0, count: 9}}, + + {&sequence{block: 0xFE, count: 8, next: &sequence{block: 0xFE, count: 1, next: &sequence{block: 0xFE, count: 5}}}, &sequence{block: 0xFE, count: 14}}, + {&sequence{block: 0xFE, count: 8, next: &sequence{block: 0xFE, count: 1, next: &sequence{block: 0xFE, count: 5, next: &sequence{block: 0xFF, count: 1}}}}, + &sequence{block: 0xFE, count: 14, next: &sequence{block: 0xFF, count: 1}}}, + + // No merge + {&sequence{block: 0xFE, count: 8, next: &sequence{block: 0xF8, count: 1, next: &sequence{block: 0xFE, count: 5}}}, + &sequence{block: 0xFE, count: 8, next: &sequence{block: 0xF8, count: 1, next: &sequence{block: 0xFE, count: 5}}}}, + + // No merge from head: // Merge function tries to merge from passed head. If it can't merge with next, it does not reattempt with next as head + {&sequence{block: 0xFE, count: 8, next: &sequence{block: 0xFF, count: 1, next: &sequence{block: 0xFF, count: 5}}}, + &sequence{block: 0xFE, count: 8, next: &sequence{block: 0xFF, count: 6}}}, + } + + for n, i := range input { + mergeSequences(i.original) + for !i.merged.equal(i.original) { + t.Fatalf("Error in (%d) mergeSequences().\nExp: %s\nGot: %s,", n, i.merged.toString(), i.original.toString()) + } + } +} + +func TestPushReservation(t *testing.T) { + input := []struct { + mask *sequence + bytePos uint32 + bitPos uint32 + newMask *sequence + }{ + // Create first sequence and fill in 8 addresses starting from address 0 + {&sequence{block: 0x0, count: 8, next: nil}, 0, 0, &sequence{block: 0x80000000, count: 1, next: &sequence{block: 0x0, count: 7, next: nil}}}, + {&sequence{block: 0x80000000, count: 8}, 0, 1, &sequence{block: 0xC0000000, count: 1, next: &sequence{block: 0x80000000, count: 7, next: nil}}}, + {&sequence{block: 0xC0000000, count: 8}, 0, 2, &sequence{block: 0xE0000000, count: 1, next: &sequence{block: 0xC0000000, count: 7, next: nil}}}, + {&sequence{block: 0xE0000000, count: 8}, 0, 3, &sequence{block: 0xF0000000, count: 1, next: &sequence{block: 0xE0000000, count: 7, next: nil}}}, + {&sequence{block: 0xF0000000, count: 8}, 0, 4, &sequence{block: 0xF8000000, count: 1, next: &sequence{block: 0xF0000000, count: 7, next: nil}}}, + {&sequence{block: 0xF8000000, count: 8}, 0, 5, &sequence{block: 0xFC000000, count: 1, next: &sequence{block: 0xF8000000, count: 7, next: nil}}}, + {&sequence{block: 0xFC000000, count: 8}, 0, 6, &sequence{block: 0xFE000000, count: 1, next: &sequence{block: 0xFC000000, count: 7, next: nil}}}, + {&sequence{block: 0xFE000000, count: 8}, 0, 7, &sequence{block: 0xFF000000, count: 1, next: &sequence{block: 0xFE000000, count: 7, next: nil}}}, + + {&sequence{block: 0x80000000, count: 1, next: &sequence{block: 0x0, count: 7}}, 0, 1, &sequence{block: 0xC0000000, count: 1, next: &sequence{block: 0x0, count: 7, next: nil}}}, + + // Create second sequence and fill in 8 addresses starting from address 32 + {&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0x00000000, count: 1, next: &sequence{block: 0xffffffff, count: 6, next: nil}}}, 4, 0, + &sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0x80000000, count: 1, next: &sequence{block: 0xffffffff, count: 6}}}}, + {&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0x80000000, count: 1, next: &sequence{block: 0xffffffff, count: 6}}}, 4, 1, + &sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0xC0000000, count: 1, next: &sequence{block: 0xffffffff, count: 6}}}}, + {&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0xC0000000, count: 1, next: &sequence{block: 0xffffffff, count: 6}}}, 4, 2, + &sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0xE0000000, count: 1, next: &sequence{block: 0xffffffff, count: 6}}}}, + {&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0xE0000000, count: 1, next: &sequence{block: 0xffffffff, count: 6}}}, 4, 3, + &sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0xF0000000, count: 1, next: &sequence{block: 0xffffffff, count: 6}}}}, + {&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0xF0000000, count: 1, next: &sequence{block: 0xffffffff, count: 6}}}, 4, 4, + &sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0xF8000000, count: 1, next: &sequence{block: 0xffffffff, count: 6}}}}, + {&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0xF8000000, count: 1, next: &sequence{block: 0xffffffff, count: 6}}}, 4, 5, + &sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0xFC000000, count: 1, next: &sequence{block: 0xffffffff, count: 6}}}}, + {&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0xFC000000, count: 1, next: &sequence{block: 0xffffffff, count: 6}}}, 4, 6, + &sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0xFE000000, count: 1, next: &sequence{block: 0xffffffff, count: 6}}}}, + {&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0xFE000000, count: 1, next: &sequence{block: 0xffffffff, count: 6}}}, 4, 7, + &sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0xFF000000, count: 1, next: &sequence{block: 0xffffffff, count: 6}}}}, + // fill in 8 addresses starting from address 40 + {&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0xFF000000, count: 1, next: &sequence{block: 0xffffffff, count: 6}}}, 5, 0, + &sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0xFF800000, count: 1, next: &sequence{block: 0xffffffff, count: 6}}}}, + {&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0xFF800000, count: 1, next: &sequence{block: 0xffffffff, count: 6}}}, 5, 1, + &sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0xFFC00000, count: 1, next: &sequence{block: 0xffffffff, count: 6}}}}, + {&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0xFFC00000, count: 1, next: &sequence{block: 0xffffffff, count: 6}}}, 5, 2, + &sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0xFFE00000, count: 1, next: &sequence{block: 0xffffffff, count: 6}}}}, + {&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0xFFE00000, count: 1, next: &sequence{block: 0xffffffff, count: 6}}}, 5, 3, + &sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0xFFF00000, count: 1, next: &sequence{block: 0xffffffff, count: 6}}}}, + {&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0xFFF00000, count: 1, next: &sequence{block: 0xffffffff, count: 6}}}, 5, 4, + &sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0xFFF80000, count: 1, next: &sequence{block: 0xffffffff, count: 6}}}}, + {&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0xFFF80000, count: 1, next: &sequence{block: 0xffffffff, count: 6}}}, 5, 5, + &sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0xFFFC0000, count: 1, next: &sequence{block: 0xffffffff, count: 6}}}}, + {&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0xFFFC0000, count: 1, next: &sequence{block: 0xffffffff, count: 6}}}, 5, 6, + &sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0xFFFE0000, count: 1, next: &sequence{block: 0xffffffff, count: 6}}}}, + {&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0xFFFE0000, count: 1, next: &sequence{block: 0xffffffff, count: 6}}}, 5, 7, + &sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0xFFFF0000, count: 1, next: &sequence{block: 0xffffffff, count: 6}}}}, + + // Insert new sequence + {&sequence{block: 0xffffffff, count: 2, next: &sequence{block: 0x0, count: 6}}, 8, 0, + &sequence{block: 0xffffffff, count: 2, next: &sequence{block: 0x80000000, count: 1, next: &sequence{block: 0x0, count: 5}}}}, + {&sequence{block: 0xffffffff, count: 2, next: &sequence{block: 0x80000000, count: 1, next: &sequence{block: 0x0, count: 5}}}, 8, 1, + &sequence{block: 0xffffffff, count: 2, next: &sequence{block: 0xC0000000, count: 1, next: &sequence{block: 0x0, count: 5}}}}, + + // Merge affected with next + {&sequence{block: 0xffffffff, count: 7, next: &sequence{block: 0xfffffffe, count: 2, next: &sequence{block: 0xffffffff, count: 1}}}, 31, 7, + &sequence{block: 0xffffffff, count: 8, next: &sequence{block: 0xfffffffe, count: 1, next: &sequence{block: 0xffffffff, count: 1}}}}, + {&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0xfffffffc, count: 1, next: &sequence{block: 0xfffffffe, count: 6}}}, 7, 6, + &sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0xfffffffe, count: 7}}}, + + // Merge affected with next and next.next + {&sequence{block: 0xffffffff, count: 7, next: &sequence{block: 0xfffffffe, count: 1, next: &sequence{block: 0xffffffff, count: 1}}}, 31, 7, + &sequence{block: 0xffffffff, count: 9}}, + {&sequence{block: 0xffffffff, count: 7, next: &sequence{block: 0xfffffffe, count: 1}}, 31, 7, + &sequence{block: 0xffffffff, count: 8}}, + + // Merge affected with previous and next + {&sequence{block: 0xffffffff, count: 7, next: &sequence{block: 0xfffffffe, count: 1, next: &sequence{block: 0xffffffff, count: 1}}}, 31, 7, + &sequence{block: 0xffffffff, count: 9}}, + + // Redundant push: No change + {&sequence{block: 0xffff0000, count: 1}, 0, 0, &sequence{block: 0xffff0000, count: 1}}, + {&sequence{block: 0xffff0000, count: 7}, 25, 7, &sequence{block: 0xffff0000, count: 7}}, + {&sequence{block: 0xffffffff, count: 7, next: &sequence{block: 0xfffffffe, count: 1, next: &sequence{block: 0xffffffff, count: 1}}}, 7, 7, + &sequence{block: 0xffffffff, count: 7, next: &sequence{block: 0xfffffffe, count: 1, next: &sequence{block: 0xffffffff, count: 1}}}}, + } + + for n, i := range input { + mask := pushReservation(i.bytePos, i.bitPos, i.mask, false) + if !mask.equal(i.newMask) { + t.Fatalf("Error in (%d) pushReservation():\n%s + (%d,%d):\nExp: %s\nGot: %s,", + n, i.mask.toString(), i.bytePos, i.bitPos, i.newMask.toString(), mask.toString()) + } + } +} + +func TestSerializeDeserialize(t *testing.T) { + s := getTestSequence() + + data, err := s.toByteArray() + if err != nil { + t.Fatal(err) + } + + r := &sequence{} + err = r.fromByteArray(data) + if err != nil { + t.Fatal(err) + } + + if !s.equal(r) { + t.Fatalf("Sequences are different: \n%v\n%v", s, r) + } +} + +func getTestSequence() *sequence { + // Returns a custom sequence of 1024 * 32 bits + return &sequence{ + block: 0XFFFFFFFF, + count: 100, + next: &sequence{ + block: 0xFFFFFFFE, + count: 1, + next: &sequence{ + block: 0xFF000000, + count: 10, + next: &sequence{ + block: 0XFFFFFFFF, + count: 50, + next: &sequence{ + block: 0XFFFFFFFC, + count: 1, + next: &sequence{ + block: 0xFF800000, + count: 1, + next: &sequence{ + block: 0XFFFFFFFF, + count: 87, + next: &sequence{ + block: 0x0, + count: 150, + next: &sequence{ + block: 0XFFFFFFFF, + count: 200, + next: &sequence{ + block: 0x0000FFFF, + count: 1, + next: &sequence{ + block: 0x0, + count: 399, + next: &sequence{ + block: 0XFFFFFFFF, + count: 23, + next: &sequence{ + block: 0x1, + count: 1, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + } +} + +func TestSet(t *testing.T) { + hnd, err := NewHandle("", nil, "", 1024*32) + if err != nil { + t.Fatal(err) + } + hnd.head = getTestSequence() + + firstAv := uint32(32*100 + 31) + last := uint32(1024*32 - 1) + + if hnd.IsSet(100000) { + t.Fatal("IsSet() returned wrong result") + } + + if !hnd.IsSet(0) { + t.Fatal("IsSet() returned wrong result") + } + + if hnd.IsSet(firstAv) { + t.Fatal("IsSet() returned wrong result") + } + + if !hnd.IsSet(last) { + t.Fatal("IsSet() returned wrong result") + } + + if err := hnd.Set(0); err == nil { + t.Fatalf("Expected failure, but succeeded") + } + + os, err := hnd.SetAny() + if err != nil { + t.Fatalf("Unexpected failure: %v", err) + } + if os != firstAv { + t.Fatalf("SetAny returned unexpected ordinal. Expected %d. Got %d.", firstAv, os) + } + if !hnd.IsSet(firstAv) { + t.Fatal("IsSet() returned wrong result") + } + + if err := hnd.Unset(firstAv); err != nil { + t.Fatalf("Unexpected failure: %v", err) + } + + if hnd.IsSet(firstAv) { + t.Fatal("IsSet() returned wrong result") + } + + if err := hnd.Set(firstAv); err != nil { + t.Fatalf("Unexpected failure: %v", err) + } + + if err := hnd.Set(last); err == nil { + t.Fatalf("Expected failure, but succeeded") + } +} + +func TestSetUnset(t *testing.T) { + numBits := uint32(64 * 1024) + hnd, err := NewHandle("", nil, "", numBits) + if err != nil { + t.Fatal(err) + } + // set and unset all one by one + for hnd.Unselected() > 0 { + hnd.SetAny() + } + i := uint32(0) + for hnd.Unselected() < numBits { + hnd.Unset(i) + i++ + } +} diff --git a/Godeps/_workspace/src/github.com/docker/libnetwork/bitseq/store.go b/Godeps/_workspace/src/github.com/docker/libnetwork/bitseq/store.go new file mode 100644 index 000000000..553f2cdf4 --- /dev/null +++ b/Godeps/_workspace/src/github.com/docker/libnetwork/bitseq/store.go @@ -0,0 +1,138 @@ +package bitseq + +import ( + "encoding/json" + "fmt" + + log "github.com/Sirupsen/logrus" + "github.com/docker/libnetwork/datastore" + "github.com/docker/libnetwork/types" +) + +// Key provides the Key to be used in KV Store +func (h *Handle) Key() []string { + h.Lock() + defer h.Unlock() + return []string{h.app, h.id} +} + +// KeyPrefix returns the immediate parent key that can be used for tree walk +func (h *Handle) KeyPrefix() []string { + h.Lock() + defer h.Unlock() + return []string{h.app} +} + +// Value marshals the data to be stored in the KV store +func (h *Handle) Value() []byte { + b, err := h.ToByteArray() + if err != nil { + log.Warnf("Failed to serialize Handle: %v", err) + b = []byte{} + } + jv, err := json.Marshal(b) + if err != nil { + log.Warnf("Failed to json encode bitseq handler byte array: %v", err) + return []byte{} + } + return jv +} + +// SetValue unmarshals the data from the KV store +func (h *Handle) SetValue(value []byte) error { + var b []byte + if err := json.Unmarshal(value, &b); err != nil { + return err + } + + return h.FromByteArray(b) +} + +// Index returns the latest DB Index as seen by this object +func (h *Handle) Index() uint64 { + h.Lock() + defer h.Unlock() + return h.dbIndex +} + +// SetIndex method allows the datastore to store the latest DB Index into this object +func (h *Handle) SetIndex(index uint64) { + h.Lock() + h.dbIndex = index + h.dbExists = true + h.Unlock() +} + +// Exists method is true if this object has been stored in the DB. +func (h *Handle) Exists() bool { + h.Lock() + defer h.Unlock() + return h.dbExists +} + +func (h *Handle) watchForChanges() error { + h.Lock() + store := h.store + h.Unlock() + + if store == nil { + return nil + } + + kvpChan, err := store.KVStore().Watch(datastore.Key(h.Key()...), nil) + if err != nil { + return err + } + go func() { + for { + select { + case kvPair := <-kvpChan: + // Only process remote update + if kvPair != nil && (kvPair.LastIndex != h.Index()) { + err := h.fromDsValue(kvPair.Value) + if err != nil { + log.Warnf("Failed to reconstruct bitseq handle from ds watch: %s", err.Error()) + } else { + h.SetIndex(kvPair.LastIndex) + } + } + } + } + }() + return nil +} + +func (h *Handle) fromDsValue(value []byte) error { + var ba []byte + if err := json.Unmarshal(value, &ba); err != nil { + return fmt.Errorf("failed to decode json: %s", err.Error()) + } + if err := h.FromByteArray(ba); err != nil { + return fmt.Errorf("failed to decode handle: %s", err.Error()) + } + return nil +} + +func (h *Handle) writeToStore() error { + h.Lock() + store := h.store + h.Unlock() + if store == nil { + return nil + } + err := store.PutObjectAtomic(h) + if err == datastore.ErrKeyModified { + return types.RetryErrorf("failed to perform atomic write (%v). Retry might fix the error", err) + } + return err +} + +func (h *Handle) deleteFromStore() error { + h.Lock() + store := h.store + h.Unlock() + if store == nil { + return nil + } + return store.DeleteObjectAtomic(h) +} diff --git a/Godeps/_workspace/src/github.com/docker/libnetwork/circle.yml b/Godeps/_workspace/src/github.com/docker/libnetwork/circle.yml new file mode 100644 index 000000000..d02f6a92e --- /dev/null +++ b/Godeps/_workspace/src/github.com/docker/libnetwork/circle.yml @@ -0,0 +1,12 @@ +machine: + services: + - docker + +dependencies: + override: + - echo "Nothing to install" + +test: + override: + - make circle-ci + diff --git a/Godeps/_workspace/src/github.com/docker/libnetwork/client/client.go b/Godeps/_workspace/src/github.com/docker/libnetwork/client/client.go new file mode 100644 index 000000000..c5713b01b --- /dev/null +++ b/Godeps/_workspace/src/github.com/docker/libnetwork/client/client.go @@ -0,0 +1,115 @@ +package client + +import ( + "fmt" + "io" + "io/ioutil" + "net/http" + "reflect" + "strings" + + flag "github.com/docker/docker/pkg/mflag" +) + +// CallFunc provides environment specific call utility to invoke backend functions from UI +type CallFunc func(string, string, interface{}, map[string][]string) (io.ReadCloser, http.Header, int, error) + +// NetworkCli is the UI object for network subcmds +type NetworkCli struct { + out io.Writer + err io.Writer + call CallFunc +} + +// NewNetworkCli is a convenient function to create a NetworkCli object +func NewNetworkCli(out, err io.Writer, call CallFunc) *NetworkCli { + return &NetworkCli{ + out: out, + err: err, + call: call, + } +} + +// getMethod is Borrowed from Docker UI which uses reflection to identify the UI Handler +func (cli *NetworkCli) getMethod(args ...string) (func(string, ...string) error, bool) { + camelArgs := make([]string, len(args)) + for i, s := range args { + if len(s) == 0 { + return nil, false + } + camelArgs[i] = strings.ToUpper(s[:1]) + strings.ToLower(s[1:]) + } + methodName := "Cmd" + strings.Join(camelArgs, "") + method := reflect.ValueOf(cli).MethodByName(methodName) + if !method.IsValid() { + return nil, false + } + return method.Interface().(func(string, ...string) error), true +} + +// Cmd is borrowed from Docker UI and acts as the entry point for network UI commands. +// network UI commands are designed to be invoked from multiple parent chains +func (cli *NetworkCli) Cmd(chain string, args ...string) error { + if len(args) > 2 { + method, exists := cli.getMethod(args[:3]...) + if exists { + return method(chain+" "+args[0]+" "+args[1], args[3:]...) + } + } + if len(args) > 1 { + method, exists := cli.getMethod(args[:2]...) + if exists { + return method(chain+" "+args[0], args[2:]...) + } + } + if len(args) > 0 { + method, exists := cli.getMethod(args[0]) + if !exists { + return fmt.Errorf("%s: '%s' is not a %s command. See '%s --help'.\n", chain, args[0], chain, chain) + } + return method(chain, args[1:]...) + } + flag.Usage() + return nil +} + +// Subcmd is borrowed from Docker UI and performs the same function of configuring the subCmds +func (cli *NetworkCli) Subcmd(chain, name, signature, description string, exitOnError bool) *flag.FlagSet { + var errorHandling flag.ErrorHandling + if exitOnError { + errorHandling = flag.ExitOnError + } else { + errorHandling = flag.ContinueOnError + } + flags := flag.NewFlagSet(name, errorHandling) + flags.Usage = func() { + flags.ShortUsage() + flags.PrintDefaults() + } + flags.ShortUsage = func() { + options := "" + if signature != "" { + signature = " " + signature + } + if flags.FlagCountUndeprecated() > 0 { + options = " [OPTIONS]" + } + fmt.Fprintf(cli.out, "\nUsage: %s %s%s%s\n\n%s\n\n", chain, name, options, signature, description) + flags.SetOutput(cli.out) + } + return flags +} + +func readBody(stream io.ReadCloser, hdr http.Header, statusCode int, err error) ([]byte, int, error) { + if stream != nil { + defer stream.Close() + } + if err != nil { + return nil, statusCode, err + } + body, err := ioutil.ReadAll(stream) + if err != nil { + return nil, -1, err + } + return body, statusCode, nil +} diff --git a/Godeps/_workspace/src/github.com/docker/libnetwork/client/client_service_test.go b/Godeps/_workspace/src/github.com/docker/libnetwork/client/client_service_test.go new file mode 100644 index 000000000..fb10fd9fd --- /dev/null +++ b/Godeps/_workspace/src/github.com/docker/libnetwork/client/client_service_test.go @@ -0,0 +1,122 @@ +package client + +import ( + "bytes" + "testing" + + _ "github.com/docker/libnetwork/netutils" +) + +func TestClientServiceInvalidCommand(t *testing.T) { + var out, errOut bytes.Buffer + cli := NewNetworkCli(&out, &errOut, callbackFunc) + + err := cli.Cmd("docker", "service", "invalid") + if err == nil { + t.Fatalf("Passing invalid commands must fail") + } +} + +func TestClientServiceCreate(t *testing.T) { + var out, errOut bytes.Buffer + cli := NewNetworkCli(&out, &errOut, callbackFunc) + + err := cli.Cmd("docker", "service", "publish", mockServiceName+"."+mockNwName) + if err != nil { + t.Fatal(err) + } +} + +func TestClientServiceRm(t *testing.T) { + var out, errOut bytes.Buffer + cli := NewNetworkCli(&out, &errOut, callbackFunc) + + err := cli.Cmd("docker", "service", "unpublish", mockServiceName+"."+mockNwName) + if err != nil { + t.Fatal(err) + } +} + +func TestClientServiceLs(t *testing.T) { + var out, errOut bytes.Buffer + cli := NewNetworkCli(&out, &errOut, callbackFunc) + + err := cli.Cmd("docker", "service", "ls") + if err != nil { + t.Fatal(err) + } +} + +func TestClientServiceInfo(t *testing.T) { + var out, errOut bytes.Buffer + cli := NewNetworkCli(&out, &errOut, callbackFunc) + + err := cli.Cmd("docker", "service", "info", mockServiceName+"."+mockNwName) + if err != nil { + t.Fatal(err) + } +} + +func TestClientServiceInfoById(t *testing.T) { + var out, errOut bytes.Buffer + cli := NewNetworkCli(&out, &errOut, callbackFunc) + + err := cli.Cmd("docker", "service", "info", mockServiceID+"."+mockNwName) + if err != nil { + t.Fatal(err) + } +} + +func TestClientServiceJoin(t *testing.T) { + var out, errOut bytes.Buffer + cli := NewNetworkCli(&out, &errOut, callbackFunc) + + err := cli.Cmd("docker", "service", "attach", mockContainerID, mockServiceName+"."+mockNwName) + if err != nil { + t.Fatal(err) + } +} + +func TestClientServiceLeave(t *testing.T) { + var out, errOut bytes.Buffer + cli := NewNetworkCli(&out, &errOut, callbackFunc) + + err := cli.Cmd("docker", "service", "detach", mockContainerID, mockServiceName+"."+mockNwName) + if err != nil { + t.Fatal(err) + } +} + +// Docker Flag processing in flag.go uses os.Exit() frequently, even for --help +// TODO : Handle the --help test-case in the IT when CLI is available +/* +func TestClientNetworkServiceCreateHelp(t *testing.T) { + var out, errOut bytes.Buffer + cFunc := func(method, path string, data interface{}, headers map[string][]string) (io.ReadCloser, int, error) { + return nil, 0, nil + } + cli := NewNetworkCli(&out, &errOut, callbackFunc) + + err := cli.Cmd("docker", "network", "create", "--help") + if err != nil { + t.Fatalf(err.Error()) + } +} +*/ + +// Docker flag processing in flag.go uses os.Exit(1) for incorrect parameter case. +// TODO : Handle the missing argument case in the IT when CLI is available +/* +func TestClientNetworkServiceCreateMissingArgument(t *testing.T) { + var out, errOut bytes.Buffer + cFunc := func(method, path string, data interface{}, headers map[string][]string) (io.ReadCloser, int, error) { + return nil, 0, nil + } + cli := NewNetworkCli(&out, &errOut, callbackFunc) + + err := cli.Cmd("docker", "network", "create") + if err != nil { + t.Fatal(err.Error()) + } +} +*/ diff --git a/Godeps/_workspace/src/github.com/docker/libnetwork/client/client_test.go b/Godeps/_workspace/src/github.com/docker/libnetwork/client/client_test.go new file mode 100644 index 000000000..06b8b2a6d --- /dev/null +++ b/Godeps/_workspace/src/github.com/docker/libnetwork/client/client_test.go @@ -0,0 +1,217 @@ +package client + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "net/http" + "os" + "strings" + "testing" + + _ "github.com/docker/libnetwork/netutils" +) + +// nopCloser is used to provide a dummy CallFunc for Cmd() +type nopCloser struct { + io.Reader +} + +func (nopCloser) Close() error { return nil } + +func TestMain(m *testing.M) { + setupMockHTTPCallback() + os.Exit(m.Run()) +} + +var callbackFunc func(method, path string, data interface{}, headers map[string][]string) (io.ReadCloser, http.Header, int, error) +var mockNwJSON, mockNwListJSON, mockServiceJSON, mockServiceListJSON []byte +var mockNwName = "test" +var mockNwID = "2a3456789" +var mockServiceName = "testSrv" +var mockServiceID = "2a3456789" +var mockContainerID = "2a3456789" + +func setupMockHTTPCallback() { + var list []networkResource + nw := networkResource{Name: mockNwName, ID: mockNwID} + mockNwJSON, _ = json.Marshal(nw) + list = append(list, nw) + mockNwListJSON, _ = json.Marshal(list) + + var srvList []serviceResource + ep := serviceResource{Name: mockServiceName, ID: mockServiceID, Network: mockNwName} + mockServiceJSON, _ = json.Marshal(ep) + srvList = append(srvList, ep) + mockServiceListJSON, _ = json.Marshal(srvList) + + dummyHTTPHdr := http.Header{} + + callbackFunc = func(method, path string, data interface{}, headers map[string][]string) (io.ReadCloser, http.Header, int, error) { + var rsp string + switch method { + case "GET": + if strings.Contains(path, fmt.Sprintf("networks?name=%s", mockNwName)) { + rsp = string(mockNwListJSON) + } else if strings.Contains(path, "networks?name=") { + rsp = "[]" + } else if strings.Contains(path, fmt.Sprintf("networks?partial-id=%s", mockNwID)) { + rsp = string(mockNwListJSON) + } else if strings.Contains(path, "networks?partial-id=") { + rsp = "[]" + } else if strings.HasSuffix(path, "networks") { + rsp = string(mockNwListJSON) + } else if strings.HasSuffix(path, "networks/"+mockNwID) { + rsp = string(mockNwJSON) + } else if strings.Contains(path, fmt.Sprintf("services?name=%s", mockServiceName)) { + rsp = string(mockServiceListJSON) + } else if strings.Contains(path, "services?name=") { + rsp = "[]" + } else if strings.Contains(path, fmt.Sprintf("services?partial-id=%s", mockServiceID)) { + rsp = string(mockServiceListJSON) + } else if strings.Contains(path, "services?partial-id=") { + rsp = "[]" + } else if strings.HasSuffix(path, "services") { + rsp = string(mockServiceListJSON) + } else if strings.HasSuffix(path, "services/"+mockServiceID) { + rsp = string(mockServiceJSON) + } else if strings.Contains(path, "containers") { + return nopCloser{bytes.NewBufferString("")}, dummyHTTPHdr, 400, fmt.Errorf("Bad Request") + } + case "POST": + var data []byte + if strings.HasSuffix(path, "networks") { + data, _ = json.Marshal(mockNwID) + } else if strings.HasSuffix(path, "services") { + data, _ = json.Marshal(mockServiceID) + } else if strings.HasSuffix(path, "backend") { + data, _ = json.Marshal(mockContainerID) + } + rsp = string(data) + case "PUT": + case "DELETE": + rsp = "" + } + return nopCloser{bytes.NewBufferString(rsp)}, dummyHTTPHdr, 200, nil + } +} + +func TestClientDummyCommand(t *testing.T) { + var out, errOut bytes.Buffer + cli := NewNetworkCli(&out, &errOut, callbackFunc) + + err := cli.Cmd("docker", "dummy") + if err == nil { + t.Fatalf("Incorrect Command must fail") + } +} + +func TestClientNetworkInvalidCommand(t *testing.T) { + var out, errOut bytes.Buffer + cli := NewNetworkCli(&out, &errOut, callbackFunc) + + err := cli.Cmd("docker", "network", "invalid") + if err == nil { + t.Fatalf("Passing invalid commands must fail") + } +} + +func TestClientNetworkCreate(t *testing.T) { + var out, errOut bytes.Buffer + cli := NewNetworkCli(&out, &errOut, callbackFunc) + + err := cli.Cmd("docker", "network", "create", mockNwName) + if err != nil { + t.Fatal(err.Error()) + } +} + +func TestClientNetworkCreateWithDriver(t *testing.T) { + var out, errOut bytes.Buffer + cli := NewNetworkCli(&out, &errOut, callbackFunc) + + err := cli.Cmd("docker", "network", "create", "-f=dummy", mockNwName) + if err == nil { + t.Fatalf("Passing incorrect flags to the create command must fail") + } + + err = cli.Cmd("docker", "network", "create", "-d=dummy", mockNwName) + if err != nil { + t.Fatalf(err.Error()) + } +} + +func TestClientNetworkRm(t *testing.T) { + var out, errOut bytes.Buffer + cli := NewNetworkCli(&out, &errOut, callbackFunc) + + err := cli.Cmd("docker", "network", "rm", mockNwName) + if err != nil { + t.Fatal(err.Error()) + } +} + +func TestClientNetworkLs(t *testing.T) { + var out, errOut bytes.Buffer + cli := NewNetworkCli(&out, &errOut, callbackFunc) + + err := cli.Cmd("docker", "network", "ls") + if err != nil { + t.Fatal(err.Error()) + } +} + +func TestClientNetworkInfo(t *testing.T) { + var out, errOut bytes.Buffer + cli := NewNetworkCli(&out, &errOut, callbackFunc) + + err := cli.Cmd("docker", "network", "info", mockNwName) + if err != nil { + t.Fatal(err.Error()) + } +} + +func TestClientNetworkInfoById(t *testing.T) { + var out, errOut bytes.Buffer + cli := NewNetworkCli(&out, &errOut, callbackFunc) + + err := cli.Cmd("docker", "network", "info", mockNwID) + if err != nil { + t.Fatal(err.Error()) + } +} + +// Docker Flag processing in flag.go uses os.Exit() frequently, even for --help +// TODO : Handle the --help test-case in the IT when CLI is available +/* +func TestClientNetworkServiceCreateHelp(t *testing.T) { + var out, errOut bytes.Buffer + cFunc := func(method, path string, data interface{}, headers map[string][]string) (io.ReadCloser, int, error) { + return nil, 0, nil + } + cli := NewNetworkCli(&out, &errOut, callbackFunc) + + err := cli.Cmd("docker", "network", "create", "--help") + if err != nil { + t.Fatalf(err.Error()) + } +} +*/ + +// Docker flag processing in flag.go uses os.Exit(1) for incorrect parameter case. +// TODO : Handle the missing argument case in the IT when CLI is available +/* +func TestClientNetworkServiceCreateMissingArgument(t *testing.T) { + var out, errOut bytes.Buffer + cFunc := func(method, path string, data interface{}, headers map[string][]string) (io.ReadCloser, int, error) { + return nil, 0, nil + } + cli := NewNetworkCli(&out, &errOut, callbackFunc) + + err := cli.Cmd("docker", "network", "create") + if err != nil { + t.Fatal(err.Error()) + } +} +*/ diff --git a/Godeps/_workspace/src/github.com/docker/libnetwork/client/network.go b/Godeps/_workspace/src/github.com/docker/libnetwork/client/network.go new file mode 100644 index 000000000..a244ad5f6 --- /dev/null +++ b/Godeps/_workspace/src/github.com/docker/libnetwork/client/network.go @@ -0,0 +1,231 @@ +package client + +import ( + "bytes" + "encoding/json" + "fmt" + "net/http" + "text/tabwriter" + + flag "github.com/docker/docker/pkg/mflag" + "github.com/docker/docker/pkg/stringid" +) + +type command struct { + name string + description string +} + +var ( + networkCommands = []command{ + {"create", "Create a network"}, + {"rm", "Remove a network"}, + {"ls", "List all networks"}, + {"info", "Display information of a network"}, + } +) + +// CmdNetwork handles the root Network UI +func (cli *NetworkCli) CmdNetwork(chain string, args ...string) error { + cmd := cli.Subcmd(chain, "network", "COMMAND [OPTIONS] [arg...]", networkUsage(chain), false) + cmd.Require(flag.Min, 1) + err := cmd.ParseFlags(args, true) + if err == nil { + cmd.Usage() + return fmt.Errorf("invalid command : %v", args) + } + return err +} + +// CmdNetworkCreate handles Network Create UI +func (cli *NetworkCli) CmdNetworkCreate(chain string, args ...string) error { + cmd := cli.Subcmd(chain, "create", "NETWORK-NAME", "Creates a new network with a name specified by the user", false) + flDriver := cmd.String([]string{"d", "-driver"}, "", "Driver to manage the Network") + cmd.Require(flag.Exact, 1) + err := cmd.ParseFlags(args, true) + if err != nil { + return err + } + + // Construct network create request body + ops := make(map[string]interface{}) + nc := networkCreate{Name: cmd.Arg(0), NetworkType: *flDriver, Options: ops} + obj, _, err := readBody(cli.call("POST", "/networks", nc, nil)) + if err != nil { + return err + } + var replyID string + err = json.Unmarshal(obj, &replyID) + if err != nil { + return err + } + fmt.Fprintf(cli.out, "%s\n", replyID) + return nil +} + +// CmdNetworkRm handles Network Delete UI +func (cli *NetworkCli) CmdNetworkRm(chain string, args ...string) error { + cmd := cli.Subcmd(chain, "rm", "NETWORK", "Deletes a network", false) + cmd.Require(flag.Exact, 1) + err := cmd.ParseFlags(args, true) + if err != nil { + return err + } + id, err := lookupNetworkID(cli, cmd.Arg(0)) + if err != nil { + return err + } + _, _, err = readBody(cli.call("DELETE", "/networks/"+id, nil, nil)) + if err != nil { + return err + } + return nil +} + +// CmdNetworkLs handles Network List UI +func (cli *NetworkCli) CmdNetworkLs(chain string, args ...string) error { + cmd := cli.Subcmd(chain, "ls", "", "Lists all the networks created by the user", false) + quiet := cmd.Bool([]string{"q", "-quiet"}, false, "Only display numeric IDs") + noTrunc := cmd.Bool([]string{"#notrunc", "-no-trunc"}, false, "Do not truncate the output") + nLatest := cmd.Bool([]string{"l", "-latest"}, false, "Show the latest network created") + last := cmd.Int([]string{"n"}, -1, "Show n last created networks") + err := cmd.ParseFlags(args, true) + if err != nil { + return err + } + obj, _, err := readBody(cli.call("GET", "/networks", nil, nil)) + if err != nil { + return err + } + if *last == -1 && *nLatest { + *last = 1 + } + + var networkResources []networkResource + err = json.Unmarshal(obj, &networkResources) + if err != nil { + return err + } + + wr := tabwriter.NewWriter(cli.out, 20, 1, 3, ' ', 0) + + // unless quiet (-q) is specified, print field titles + if !*quiet { + fmt.Fprintln(wr, "NETWORK ID\tNAME\tTYPE") + } + + for _, networkResource := range networkResources { + ID := networkResource.ID + netName := networkResource.Name + if !*noTrunc { + ID = stringid.TruncateID(ID) + } + if *quiet { + fmt.Fprintln(wr, ID) + continue + } + netType := networkResource.Type + fmt.Fprintf(wr, "%s\t%s\t%s\t", + ID, + netName, + netType) + fmt.Fprint(wr, "\n") + } + wr.Flush() + return nil +} + +// CmdNetworkInfo handles Network Info UI +func (cli *NetworkCli) CmdNetworkInfo(chain string, args ...string) error { + cmd := cli.Subcmd(chain, "info", "NETWORK", "Displays detailed information on a network", false) + cmd.Require(flag.Exact, 1) + err := cmd.ParseFlags(args, true) + if err != nil { + return err + } + + id, err := lookupNetworkID(cli, cmd.Arg(0)) + if err != nil { + return err + } + + obj, _, err := readBody(cli.call("GET", "/networks/"+id, nil, nil)) + if err != nil { + return err + } + networkResource := &networkResource{} + if err := json.NewDecoder(bytes.NewReader(obj)).Decode(networkResource); err != nil { + return err + } + fmt.Fprintf(cli.out, "Network Id: %s\n", networkResource.ID) + fmt.Fprintf(cli.out, "Name: %s\n", networkResource.Name) + fmt.Fprintf(cli.out, "Type: %s\n", networkResource.Type) + if networkResource.Services != nil { + for _, serviceResource := range networkResource.Services { + fmt.Fprintf(cli.out, " Service Id: %s\n", serviceResource.ID) + fmt.Fprintf(cli.out, "\tName: %s\n", serviceResource.Name) + } + } + + return nil +} + +// Helper function to predict if a string is a name or id or partial-id +// This provides a best-effort mechanism to identify a id with the help of GET Filter APIs +// Being a UI, its most likely that name will be used by the user, which is used to lookup +// the corresponding ID. If ID is not found, this function will assume that the passed string +// is an ID by itself. + +func lookupNetworkID(cli *NetworkCli, nameID string) (string, error) { + obj, statusCode, err := readBody(cli.call("GET", "/networks?name="+nameID, nil, nil)) + if err != nil { + return "", err + } + + if statusCode != http.StatusOK { + return "", fmt.Errorf("name query failed for %s due to : statuscode(%d) %v", nameID, statusCode, string(obj)) + } + + var list []*networkResource + err = json.Unmarshal(obj, &list) + if err != nil { + return "", err + } + if len(list) > 0 { + // name query filter will always return a single-element collection + return list[0].ID, nil + } + + // Check for Partial-id + obj, statusCode, err = readBody(cli.call("GET", "/networks?partial-id="+nameID, nil, nil)) + if err != nil { + return "", err + } + + if statusCode != http.StatusOK { + return "", fmt.Errorf("partial-id match query failed for %s due to : statuscode(%d) %v", nameID, statusCode, string(obj)) + } + + err = json.Unmarshal(obj, &list) + if err != nil { + return "", err + } + if len(list) == 0 { + return "", fmt.Errorf("resource not found %s", nameID) + } + if len(list) > 1 { + return "", fmt.Errorf("multiple Networks matching the partial identifier (%s). Please use full identifier", nameID) + } + return list[0].ID, nil +} + +func networkUsage(chain string) string { + help := "Commands:\n" + + for _, cmd := range networkCommands { + help += fmt.Sprintf(" %-25.25s%s\n", cmd.name, cmd.description) + } + + help += fmt.Sprintf("\nRun '%s network COMMAND --help' for more information on a command.", chain) + return help +} diff --git a/Godeps/_workspace/src/github.com/docker/libnetwork/client/service.go b/Godeps/_workspace/src/github.com/docker/libnetwork/client/service.go new file mode 100644 index 000000000..35e040f80 --- /dev/null +++ b/Godeps/_workspace/src/github.com/docker/libnetwork/client/service.go @@ -0,0 +1,362 @@ +package client + +import ( + "bytes" + "encoding/json" + "fmt" + "net/http" + "strings" + "text/tabwriter" + + flag "github.com/docker/docker/pkg/mflag" + "github.com/docker/docker/pkg/stringid" +) + +var ( + serviceCommands = []command{ + {"publish", "Publish a service"}, + {"unpublish", "Remove a service"}, + {"attach", "Attach a backend (container) to the service"}, + {"detach", "Detach the backend from the service"}, + {"ls", "Lists all services"}, + {"info", "Display information about a service"}, + } +) + +func lookupServiceID(cli *NetworkCli, nwName, svNameID string) (string, error) { + // Sanity Check + obj, _, err := readBody(cli.call("GET", fmt.Sprintf("/networks?name=%s", nwName), nil, nil)) + if err != nil { + return "", err + } + var nwList []networkResource + if err = json.Unmarshal(obj, &nwList); err != nil { + return "", err + } + if len(nwList) == 0 { + return "", fmt.Errorf("Network %s does not exist", nwName) + } + + if nwName == "" { + obj, _, err := readBody(cli.call("GET", "/networks/"+nwList[0].ID, nil, nil)) + if err != nil { + return "", err + } + networkResource := &networkResource{} + if err := json.NewDecoder(bytes.NewReader(obj)).Decode(networkResource); err != nil { + return "", err + } + nwName = networkResource.Name + } + + // Query service by name + obj, statusCode, err := readBody(cli.call("GET", fmt.Sprintf("/services?name=%s", svNameID), nil, nil)) + if err != nil { + return "", err + } + + if statusCode != http.StatusOK { + return "", fmt.Errorf("name query failed for %s due to: (%d) %s", svNameID, statusCode, string(obj)) + } + + var list []*serviceResource + if err = json.Unmarshal(obj, &list); err != nil { + return "", err + } + for _, sr := range list { + if sr.Network == nwName { + return sr.ID, nil + } + } + + // Query service by Partial-id (this covers full id as well) + obj, statusCode, err = readBody(cli.call("GET", fmt.Sprintf("/services?partial-id=%s", svNameID), nil, nil)) + if err != nil { + return "", err + } + + if statusCode != http.StatusOK { + return "", fmt.Errorf("partial-id match query failed for %s due to: (%d) %s", svNameID, statusCode, string(obj)) + } + + if err = json.Unmarshal(obj, &list); err != nil { + return "", err + } + for _, sr := range list { + if sr.Network == nwName { + return sr.ID, nil + } + } + + return "", fmt.Errorf("Service %s not found on network %s", svNameID, nwName) +} + +func lookupContainerID(cli *NetworkCli, cnNameID string) (string, error) { + // Container is a Docker resource, ask docker about it. + // In case of connecton error, we assume we are running in dnet and return whatever was passed to us + obj, _, err := readBody(cli.call("GET", fmt.Sprintf("/containers/%s/json", cnNameID), nil, nil)) + if err != nil { + // We are probably running outside of docker + return cnNameID, nil + } + + var x map[string]interface{} + err = json.Unmarshal(obj, &x) + if err != nil { + return "", err + } + if iid, ok := x["Id"]; ok { + if id, ok := iid.(string); ok { + return id, nil + } + return "", fmt.Errorf("Unexpected data type for container ID in json response") + } + return "", fmt.Errorf("Cannot find container ID in json response") +} + +// CmdService handles the service UI +func (cli *NetworkCli) CmdService(chain string, args ...string) error { + cmd := cli.Subcmd(chain, "service", "COMMAND [OPTIONS] [arg...]", serviceUsage(chain), false) + cmd.Require(flag.Min, 1) + err := cmd.ParseFlags(args, true) + if err == nil { + cmd.Usage() + return fmt.Errorf("Invalid command : %v", args) + } + return err +} + +// Parse service name for "SERVICE[.NETWORK]" format +func parseServiceName(name string) (string, string) { + s := strings.Split(name, ".") + var sName, nName string + if len(s) > 1 { + nName = s[len(s)-1] + sName = strings.Join(s[:len(s)-1], ".") + } else { + sName = s[0] + } + return sName, nName +} + +// CmdServicePublish handles service create UI +func (cli *NetworkCli) CmdServicePublish(chain string, args ...string) error { + cmd := cli.Subcmd(chain, "publish", "SERVICE[.NETWORK]", "Publish a new service on a network", false) + cmd.Require(flag.Exact, 1) + err := cmd.ParseFlags(args, true) + if err != nil { + return err + } + + sn, nn := parseServiceName(cmd.Arg(0)) + sc := serviceCreate{Name: sn, Network: nn} + obj, _, err := readBody(cli.call("POST", "/services", sc, nil)) + if err != nil { + return err + } + + var replyID string + err = json.Unmarshal(obj, &replyID) + if err != nil { + return err + } + + fmt.Fprintf(cli.out, "%s\n", replyID) + return nil +} + +// CmdServiceUnpublish handles service delete UI +func (cli *NetworkCli) CmdServiceUnpublish(chain string, args ...string) error { + cmd := cli.Subcmd(chain, "unpublish", "SERVICE[.NETWORK]", "Removes a service", false) + cmd.Require(flag.Exact, 1) + err := cmd.ParseFlags(args, true) + if err != nil { + return err + } + + sn, nn := parseServiceName(cmd.Arg(0)) + serviceID, err := lookupServiceID(cli, nn, sn) + if err != nil { + return err + } + + _, _, err = readBody(cli.call("DELETE", "/services/"+serviceID, nil, nil)) + + return err +} + +// CmdServiceLs handles service list UI +func (cli *NetworkCli) CmdServiceLs(chain string, args ...string) error { + cmd := cli.Subcmd(chain, "ls", "SERVICE", "Lists all the services on a network", false) + flNetwork := cmd.String([]string{"net", "-network"}, "", "Only show the services that are published on the specified network") + quiet := cmd.Bool([]string{"q", "-quiet"}, false, "Only display numeric IDs") + noTrunc := cmd.Bool([]string{"#notrunc", "-no-trunc"}, false, "Do not truncate the output") + + err := cmd.ParseFlags(args, true) + if err != nil { + return err + } + + var obj []byte + if *flNetwork == "" { + obj, _, err = readBody(cli.call("GET", "/services", nil, nil)) + } else { + obj, _, err = readBody(cli.call("GET", "/services?network="+*flNetwork, nil, nil)) + } + if err != nil { + return err + } + + var serviceResources []serviceResource + err = json.Unmarshal(obj, &serviceResources) + if err != nil { + fmt.Println(err) + return err + } + + wr := tabwriter.NewWriter(cli.out, 20, 1, 3, ' ', 0) + // unless quiet (-q) is specified, print field titles + if !*quiet { + fmt.Fprintln(wr, "SERVICE ID\tNAME\tNETWORK\tCONTAINER") + } + + for _, sr := range serviceResources { + ID := sr.ID + bkID, err := getBackendID(cli, ID) + if err != nil { + return err + } + if !*noTrunc { + ID = stringid.TruncateID(ID) + bkID = stringid.TruncateID(bkID) + } + if !*quiet { + fmt.Fprintf(wr, "%s\t%s\t%s\t%s\n", ID, sr.Name, sr.Network, bkID) + } else { + fmt.Fprintln(wr, ID) + } + } + wr.Flush() + + return nil +} + +func getBackendID(cli *NetworkCli, servID string) (string, error) { + var ( + obj []byte + err error + bk string + ) + + if obj, _, err = readBody(cli.call("GET", "/services/"+servID+"/backend", nil, nil)); err == nil { + var bkl []backendResource + if err := json.NewDecoder(bytes.NewReader(obj)).Decode(&bkl); err == nil { + if len(bkl) > 0 { + bk = bkl[0].ID + } + } else { + // Only print a message, don't make the caller cli fail for this + fmt.Fprintf(cli.out, "Failed to retrieve backend list for service %s (%v)", servID, err) + } + } + + return bk, err +} + +// CmdServiceInfo handles service info UI +func (cli *NetworkCli) CmdServiceInfo(chain string, args ...string) error { + cmd := cli.Subcmd(chain, "info", "SERVICE[.NETWORK]", "Displays detailed information about a service", false) + cmd.Require(flag.Min, 1) + + err := cmd.ParseFlags(args, true) + if err != nil { + return err + } + + sn, nn := parseServiceName(cmd.Arg(0)) + serviceID, err := lookupServiceID(cli, nn, sn) + if err != nil { + return err + } + + obj, _, err := readBody(cli.call("GET", "/services/"+serviceID, nil, nil)) + if err != nil { + return err + } + + sr := &serviceResource{} + if err := json.NewDecoder(bytes.NewReader(obj)).Decode(sr); err != nil { + return err + } + + fmt.Fprintf(cli.out, "Service Id: %s\n", sr.ID) + fmt.Fprintf(cli.out, "\tName: %s\n", sr.Name) + fmt.Fprintf(cli.out, "\tNetwork: %s\n", sr.Network) + + return nil +} + +// CmdServiceAttach handles service attach UI +func (cli *NetworkCli) CmdServiceAttach(chain string, args ...string) error { + cmd := cli.Subcmd(chain, "attach", "CONTAINER SERVICE[.NETWORK]", "Sets a container as a service backend", false) + cmd.Require(flag.Min, 2) + err := cmd.ParseFlags(args, true) + if err != nil { + return err + } + + containerID, err := lookupContainerID(cli, cmd.Arg(0)) + if err != nil { + return err + } + + sn, nn := parseServiceName(cmd.Arg(1)) + serviceID, err := lookupServiceID(cli, nn, sn) + if err != nil { + return err + } + + nc := serviceAttach{ContainerID: containerID} + + _, _, err = readBody(cli.call("POST", "/services/"+serviceID+"/backend", nc, nil)) + + return err +} + +// CmdServiceDetach handles service detach UI +func (cli *NetworkCli) CmdServiceDetach(chain string, args ...string) error { + cmd := cli.Subcmd(chain, "detach", "CONTAINER SERVICE", "Removes a container from service backend", false) + cmd.Require(flag.Min, 2) + err := cmd.ParseFlags(args, true) + if err != nil { + return err + } + + sn, nn := parseServiceName(cmd.Arg(1)) + containerID, err := lookupContainerID(cli, cmd.Arg(0)) + if err != nil { + return err + } + + serviceID, err := lookupServiceID(cli, nn, sn) + if err != nil { + return err + } + + _, _, err = readBody(cli.call("DELETE", "/services/"+serviceID+"/backend/"+containerID, nil, nil)) + if err != nil { + return err + } + return nil +} + +func serviceUsage(chain string) string { + help := "Commands:\n" + + for _, cmd := range serviceCommands { + help += fmt.Sprintf(" %-10.10s%s\n", cmd.name, cmd.description) + } + + help += fmt.Sprintf("\nRun '%s service COMMAND --help' for more information on a command.", chain) + return help +} diff --git a/Godeps/_workspace/src/github.com/docker/libnetwork/client/types.go b/Godeps/_workspace/src/github.com/docker/libnetwork/client/types.go new file mode 100644 index 000000000..e14460aba --- /dev/null +++ b/Godeps/_workspace/src/github.com/docker/libnetwork/client/types.go @@ -0,0 +1,73 @@ +package client + +import "github.com/docker/libnetwork/types" + +/*********** + Resources +************/ + +// networkResource is the body of the "get network" http response message +type networkResource struct { + Name string `json:"name"` + ID string `json:"id"` + Type string `json:"type"` + Services []*serviceResource `json:"services"` +} + +// serviceResource is the body of the "get service" http response message +type serviceResource struct { + Name string `json:"name"` + ID string `json:"id"` + Network string `json:"network"` +} + +// backendResource is the body of "get service backend" response message +type backendResource struct { + ID string `json:"id"` +} + +/*********** + Body types + ************/ + +// networkCreate is the expected body of the "create network" http request message +type networkCreate struct { + Name string `json:"name"` + NetworkType string `json:"network_type"` + Options map[string]interface{} `json:"options"` +} + +// serviceCreate represents the body of the "publish service" http request message +type serviceCreate struct { + Name string `json:"name"` + Network string `json:"network_name"` + ExposedPorts []types.TransportPort `json:"exposed_ports"` + PortMapping []types.PortBinding `json:"port_mapping"` +} + +// serviceAttach represents the expected body of the "attach/detach backend to/from service" http request messages +type serviceAttach struct { + ContainerID string `json:"container_id"` + HostName string `json:"host_name"` + DomainName string `json:"domain_name"` + HostsPath string `json:"hosts_path"` + ResolvConfPath string `json:"resolv_conf_path"` + DNS []string `json:"dns"` + ExtraHosts []serviceExtraHost `json:"extra_hosts"` + ParentUpdates []serviceParentUpdate `json:"parent_updates"` + UseDefaultSandbox bool `json:"use_default_sandbox"` +} + +// serviceExtraHost represents the extra host object +type serviceExtraHost struct { + Name string `json:"name"` + Address string `json:"address"` +} + +// EndpointParentUpdate is the object carrying the information about the +// endpoint parent that needs to be updated +type serviceParentUpdate struct { + EndpointID string `json:"service_id"` + Name string `json:"name"` + Address string `json:"address"` +} diff --git a/Godeps/_workspace/src/github.com/docker/libnetwork/cmd/dnet/dnet.go b/Godeps/_workspace/src/github.com/docker/libnetwork/cmd/dnet/dnet.go new file mode 100644 index 000000000..c0d497d4c --- /dev/null +++ b/Godeps/_workspace/src/github.com/docker/libnetwork/cmd/dnet/dnet.go @@ -0,0 +1,288 @@ +package main + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "io/ioutil" + "net/http" + "os" + "strings" + + flag "github.com/docker/docker/pkg/mflag" + "github.com/docker/docker/pkg/parsers" + "github.com/docker/docker/pkg/reexec" + + "github.com/Sirupsen/logrus" + "github.com/docker/docker/pkg/term" + "github.com/docker/libnetwork" + "github.com/docker/libnetwork/api" + "github.com/docker/libnetwork/client" + "github.com/docker/libnetwork/config" + "github.com/docker/libnetwork/netlabel" + "github.com/docker/libnetwork/options" + "github.com/gorilla/mux" +) + +const ( + // DefaultHTTPHost is used if only port is provided to -H flag e.g. docker -d -H tcp://:8080 + DefaultHTTPHost = "127.0.0.1" + // DefaultHTTPPort is the default http port used by dnet + DefaultHTTPPort = 2385 + // DefaultUnixSocket exported + DefaultUnixSocket = "/var/run/dnet.sock" + cfgFileEnv = "LIBNETWORK_CFG" + defaultCfgFile = "/etc/default/libnetwork.toml" +) + +func main() { + if reexec.Init() { + return + } + + _, stdout, stderr := term.StdStreams() + logrus.SetOutput(stderr) + + err := dnetCommand(stdout, stderr) + if err != nil { + os.Exit(1) + } +} + +func parseConfig(cfgFile string) (*config.Config, error) { + if strings.Trim(cfgFile, " ") == "" { + cfgFile = os.Getenv(cfgFileEnv) + if strings.Trim(cfgFile, " ") == "" { + cfgFile = defaultCfgFile + } + } + return config.ParseConfig(cfgFile) +} + +func processConfig(cfg *config.Config) []config.Option { + options := []config.Option{} + if cfg == nil { + return options + } + dn := "bridge" + if strings.TrimSpace(cfg.Daemon.DefaultNetwork) != "" { + dn = cfg.Daemon.DefaultNetwork + } + options = append(options, config.OptionDefaultNetwork(dn)) + + dd := "bridge" + if strings.TrimSpace(cfg.Daemon.DefaultDriver) != "" { + dd = cfg.Daemon.DefaultDriver + } + options = append(options, config.OptionDefaultDriver(dd)) + + if cfg.Daemon.Labels != nil { + options = append(options, config.OptionLabels(cfg.Daemon.Labels)) + } + if strings.TrimSpace(cfg.Datastore.Client.Provider) != "" { + options = append(options, config.OptionKVProvider(cfg.Datastore.Client.Provider)) + } + if strings.TrimSpace(cfg.Datastore.Client.Address) != "" { + options = append(options, config.OptionKVProviderURL(cfg.Datastore.Client.Address)) + } + return options +} + +func dnetCommand(stdout, stderr io.Writer) error { + flag.Parse() + + if *flHelp { + flag.Usage() + return nil + } + + if *flLogLevel != "" { + lvl, err := logrus.ParseLevel(*flLogLevel) + if err != nil { + fmt.Fprintf(stderr, "Unable to parse logging level: %s\n", *flLogLevel) + return err + } + logrus.SetLevel(lvl) + } else { + logrus.SetLevel(logrus.InfoLevel) + } + + if *flDebug { + logrus.SetLevel(logrus.DebugLevel) + } + + if *flHost == "" { + defaultHost := os.Getenv("DNET_HOST") + if defaultHost == "" { + // TODO : Add UDS support + defaultHost = fmt.Sprintf("tcp://%s:%d", DefaultHTTPHost, DefaultHTTPPort) + } + *flHost = defaultHost + } + + dc, err := newDnetConnection(*flHost) + if err != nil { + if *flDaemon { + logrus.Error(err) + } else { + fmt.Fprint(stderr, err) + } + return err + } + + if *flDaemon { + err := dc.dnetDaemon() + if err != nil { + logrus.Errorf("dnet Daemon exited with an error : %v", err) + } + return err + } + + cli := client.NewNetworkCli(stdout, stderr, dc.httpCall) + if err := cli.Cmd("dnet", flag.Args()...); err != nil { + fmt.Fprintln(stderr, err) + return err + } + return nil +} + +func createDefaultNetwork(c libnetwork.NetworkController) { + nw := c.Config().Daemon.DefaultNetwork + d := c.Config().Daemon.DefaultDriver + createOptions := []libnetwork.NetworkOption{} + genericOption := options.Generic{} + + if nw != "" && d != "" { + // Bridge driver is special due to legacy reasons + if d == "bridge" { + genericOption[netlabel.GenericData] = map[string]interface{}{ + "BridgeName": nw, + "AllowNonDefaultBridge": "true", + } + networkOption := libnetwork.NetworkOptionGeneric(genericOption) + createOptions = append(createOptions, networkOption) + } + _, err := c.NewNetwork(d, nw, createOptions...) + if err != nil { + logrus.Errorf("Error creating default network : %s : %v", nw, err) + } + } +} + +type dnetConnection struct { + // proto holds the client protocol i.e. unix. + proto string + // addr holds the client address. + addr string +} + +func (d *dnetConnection) dnetDaemon() error { + cfg, err := parseConfig(*flCfgFile) + var cOptions []config.Option + if err == nil { + cOptions = processConfig(cfg) + } + controller, err := libnetwork.New(cOptions...) + if err != nil { + fmt.Println("Error starting dnetDaemon :", err) + return err + } + createDefaultNetwork(controller) + httpHandler := api.NewHTTPHandler(controller) + r := mux.NewRouter().StrictSlash(false) + post := r.PathPrefix("/{.*}/networks").Subrouter() + post.Methods("GET", "PUT", "POST", "DELETE").HandlerFunc(httpHandler) + post = r.PathPrefix("/networks").Subrouter() + post.Methods("GET", "PUT", "POST", "DELETE").HandlerFunc(httpHandler) + post = r.PathPrefix("/{.*}/services").Subrouter() + post.Methods("GET", "PUT", "POST", "DELETE").HandlerFunc(httpHandler) + post = r.PathPrefix("/services").Subrouter() + post.Methods("GET", "PUT", "POST", "DELETE").HandlerFunc(httpHandler) + return http.ListenAndServe(d.addr, r) +} + +func newDnetConnection(val string) (*dnetConnection, error) { + url, err := parsers.ParseHost(DefaultHTTPHost, DefaultUnixSocket, val) + if err != nil { + return nil, err + } + protoAddrParts := strings.SplitN(url, "://", 2) + if len(protoAddrParts) != 2 { + return nil, fmt.Errorf("bad format, expected tcp://ADDR") + } + if strings.ToLower(protoAddrParts[0]) != "tcp" { + return nil, fmt.Errorf("dnet currently only supports tcp transport") + } + + return &dnetConnection{protoAddrParts[0], protoAddrParts[1]}, nil +} + +func (d *dnetConnection) httpCall(method, path string, data interface{}, headers map[string][]string) (io.ReadCloser, http.Header, int, error) { + var in io.Reader + in, err := encodeData(data) + if err != nil { + return nil, nil, -1, err + } + + req, err := http.NewRequest(method, fmt.Sprintf("%s", path), in) + if err != nil { + return nil, nil, -1, err + } + + setupRequestHeaders(method, data, req, headers) + + req.URL.Host = d.addr + req.URL.Scheme = "http" + + httpClient := &http.Client{} + resp, err := httpClient.Do(req) + statusCode := -1 + if resp != nil { + statusCode = resp.StatusCode + } + if err != nil { + return nil, nil, statusCode, fmt.Errorf("error when trying to connect: %v", err) + } + + if statusCode < 200 || statusCode >= 400 { + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return nil, nil, statusCode, err + } + return nil, nil, statusCode, fmt.Errorf("error : %s", bytes.TrimSpace(body)) + } + + return resp.Body, resp.Header, statusCode, nil +} + +func setupRequestHeaders(method string, data interface{}, req *http.Request, headers map[string][]string) { + if data != nil { + if headers == nil { + headers = make(map[string][]string) + } + headers["Content-Type"] = []string{"application/json"} + } + + expectedPayload := (method == "POST" || method == "PUT") + + if expectedPayload && req.Header.Get("Content-Type") == "" { + req.Header.Set("Content-Type", "text/plain") + } + + if headers != nil { + for k, v := range headers { + req.Header[k] = v + } + } +} + +func encodeData(data interface{}) (*bytes.Buffer, error) { + params := bytes.NewBuffer(nil) + if data != nil { + if err := json.NewEncoder(params).Encode(data); err != nil { + return nil, err + } + } + return params, nil +} diff --git a/Godeps/_workspace/src/github.com/docker/libnetwork/cmd/dnet/dnet_test.go b/Godeps/_workspace/src/github.com/docker/libnetwork/cmd/dnet/dnet_test.go new file mode 100644 index 000000000..b8466f1a7 --- /dev/null +++ b/Godeps/_workspace/src/github.com/docker/libnetwork/cmd/dnet/dnet_test.go @@ -0,0 +1,132 @@ +package main + +import ( + "fmt" + "io/ioutil" + "os" + "testing" + "time" + + "github.com/docker/libnetwork/netutils" +) + +const dnetCommandName = "dnet" + +var origStdOut = os.Stdout + +func TestDnetDaemonCustom(t *testing.T) { + if !netutils.IsRunningInContainer() { + t.Skip("This test must run inside a container ") + } + customPort := 4567 + doneChan := make(chan bool) + go func() { + args := []string{dnetCommandName, "-d", fmt.Sprintf("-H=:%d", customPort)} + executeDnetCommand(t, args, true) + doneChan <- true + }() + + select { + case <-doneChan: + t.Fatal("dnet Daemon is not supposed to exit") + case <-time.After(3 * time.Second): + args := []string{dnetCommandName, "-d=false", fmt.Sprintf("-H=:%d", customPort), "-D", "network", "ls"} + executeDnetCommand(t, args, true) + } +} + +func TestDnetDaemonInvalidCustom(t *testing.T) { + if !netutils.IsRunningInContainer() { + t.Skip("This test must run inside a container ") + } + customPort := 4668 + doneChan := make(chan bool) + go func() { + args := []string{dnetCommandName, "-d=true", fmt.Sprintf("-H=:%d", customPort)} + executeDnetCommand(t, args, true) + doneChan <- true + }() + + select { + case <-doneChan: + t.Fatal("dnet Daemon is not supposed to exit") + case <-time.After(3 * time.Second): + args := []string{dnetCommandName, "-d=false", "-H=:6669", "-D", "network", "ls"} + executeDnetCommand(t, args, false) + } +} + +func TestDnetDaemonInvalidParams(t *testing.T) { + if !netutils.IsRunningInContainer() { + t.Skip("This test must run inside a container ") + } + args := []string{dnetCommandName, "-d=false", "-H=tcp:/127.0.0.1:8080"} + executeDnetCommand(t, args, false) + + args = []string{dnetCommandName, "-d=false", "-H=unix://var/run/dnet.sock"} + executeDnetCommand(t, args, false) + + args = []string{dnetCommandName, "-d=false", "-H=", "-l=invalid"} + executeDnetCommand(t, args, false) + + args = []string{dnetCommandName, "-d=false", "-H=", "-l=error", "invalid"} + executeDnetCommand(t, args, false) +} + +func TestDnetDefaultsWithFlags(t *testing.T) { + if !netutils.IsRunningInContainer() { + t.Skip("This test must run inside a container ") + } + doneChan := make(chan bool) + go func() { + args := []string{dnetCommandName, "-d=true", "-H=", "-l=error"} + executeDnetCommand(t, args, true) + doneChan <- true + }() + + select { + case <-doneChan: + t.Fatal("dnet Daemon is not supposed to exit") + case <-time.After(3 * time.Second): + args := []string{dnetCommandName, "-d=false", "network", "create", "-d=null", "test"} + executeDnetCommand(t, args, true) + + args = []string{dnetCommandName, "-d=false", "-D", "network", "ls"} + executeDnetCommand(t, args, true) + } +} + +func TestDnetMain(t *testing.T) { + if !netutils.IsRunningInContainer() { + t.Skip("This test must run inside a container ") + } + customPort := 4568 + doneChan := make(chan bool) + go func() { + args := []string{dnetCommandName, "-d=true", "-h=false", fmt.Sprintf("-H=:%d", customPort)} + os.Args = args + main() + doneChan <- true + }() + select { + case <-doneChan: + t.Fatal("dnet Daemon is not supposed to exit") + case <-time.After(2 * time.Second): + } +} + +func executeDnetCommand(t *testing.T, args []string, shouldSucced bool) { + _, w, _ := os.Pipe() + os.Stdout = w + + os.Args = args + err := dnetCommand(ioutil.Discard, ioutil.Discard) + if shouldSucced && err != nil { + os.Stdout = origStdOut + t.Fatalf("cli [%v] must succeed, but failed with an error : %v", args, err) + } else if !shouldSucced && err == nil { + os.Stdout = origStdOut + t.Fatalf("cli [%v] must fail, but succeeded with an error : %v", args, err) + } + os.Stdout = origStdOut +} diff --git a/Godeps/_workspace/src/github.com/docker/libnetwork/cmd/dnet/flags.go b/Godeps/_workspace/src/github.com/docker/libnetwork/cmd/dnet/flags.go new file mode 100644 index 000000000..27dfbd19e --- /dev/null +++ b/Godeps/_workspace/src/github.com/docker/libnetwork/cmd/dnet/flags.go @@ -0,0 +1,51 @@ +package main + +import ( + "fmt" + "os" + + flag "github.com/docker/docker/pkg/mflag" +) + +type command struct { + name string + description string +} + +type byName []command + +var ( + flDaemon = flag.Bool([]string{"d", "-daemon"}, false, "Enable daemon mode") + flHost = flag.String([]string{"H", "-host"}, "", "Daemon socket to connect to") + flLogLevel = flag.String([]string{"l", "-log-level"}, "info", "Set the logging level") + flDebug = flag.Bool([]string{"D", "-debug"}, false, "Enable debug mode") + flCfgFile = flag.String([]string{"c", "-cfg-file"}, "/etc/default/libnetwork.toml", "Configuration file") + flHelp = flag.Bool([]string{"h", "-help"}, false, "Print usage") + + dnetCommands = []command{ + {"network", "Network management commands"}, + {"service", "Service management commands"}, + } +) + +func init() { + flag.Usage = func() { + fmt.Fprint(os.Stdout, "Usage: dnet [OPTIONS] COMMAND [arg...]\n\nA self-sufficient runtime for container networking.\n\nOptions:\n") + + flag.CommandLine.SetOutput(os.Stdout) + flag.PrintDefaults() + + help := "\nCommands:\n" + + for _, cmd := range dnetCommands { + help += fmt.Sprintf(" %-10.10s%s\n", cmd.name, cmd.description) + } + + help += "\nRun 'dnet COMMAND --help' for more information on a command." + fmt.Fprintf(os.Stdout, "%s\n", help) + } +} + +func printUsage() { + fmt.Println("Usage: dnet COMMAND [arg...]") +} diff --git a/Godeps/_workspace/src/github.com/docker/libnetwork/cmd/dnet/libnetwork.toml b/Godeps/_workspace/src/github.com/docker/libnetwork/cmd/dnet/libnetwork.toml new file mode 100644 index 000000000..4e22516d1 --- /dev/null +++ b/Godeps/_workspace/src/github.com/docker/libnetwork/cmd/dnet/libnetwork.toml @@ -0,0 +1,12 @@ +title = "LibNetwork Configuration file" + +[daemon] + debug = false +[cluster] + discovery = "token://22aa23948f4f6b31230687689636959e" + Address = "1.1.1.1" +[datastore] + embedded = false +[datastore.client] + provider = "consul" + Address = "localhost:8500" diff --git a/Godeps/_workspace/src/github.com/docker/libnetwork/cmd/ovrouter/ovrouter.go b/Godeps/_workspace/src/github.com/docker/libnetwork/cmd/ovrouter/ovrouter.go new file mode 100644 index 000000000..a75b55067 --- /dev/null +++ b/Godeps/_workspace/src/github.com/docker/libnetwork/cmd/ovrouter/ovrouter.go @@ -0,0 +1,149 @@ +package main + +import ( + "fmt" + "net" + "os" + "os/signal" + + "github.com/docker/docker/pkg/reexec" + "github.com/docker/libnetwork/driverapi" + "github.com/docker/libnetwork/drivers/overlay" + "github.com/docker/libnetwork/netlabel" + "github.com/docker/libnetwork/types" + "github.com/vishvananda/netlink" +) + +type router struct { + d driverapi.Driver +} + +type endpoint struct { + addr net.IPNet + mac net.HardwareAddr + name string + id int +} + +func (r *router) RegisterDriver(name string, driver driverapi.Driver, c driverapi.Capability) error { + r.d = driver + return nil +} + +func (ep *endpoint) Interfaces() []driverapi.InterfaceInfo { + return nil +} + +func (ep *endpoint) AddInterface(ID int, mac net.HardwareAddr, ipv4 net.IPNet, ipv6 net.IPNet) error { + ep.id = ID + ep.addr = ipv4 + ep.mac = mac + return nil +} + +func (ep *endpoint) InterfaceNames() []driverapi.InterfaceNameInfo { + return []driverapi.InterfaceNameInfo{ep} + +} + +func (ep *endpoint) SetNames(srcName, dstPrefix string) error { + ep.name = srcName + return nil +} + +func (ep *endpoint) ID() int { + return ep.id +} + +func (ep *endpoint) SetGateway(net.IP) error { + return nil +} + +func (ep *endpoint) SetGatewayIPv6(net.IP) error { + return nil +} + +func (ep *endpoint) AddStaticRoute(destination *net.IPNet, routeType int, + nextHop net.IP, interfaceID int) error { + return nil +} + +func (ep *endpoint) SetHostsPath(string) error { + return nil +} + +func (ep *endpoint) SetResolvConfPath(string) error { + return nil +} + +func main() { + if reexec.Init() { + return + } + + r := &router{} + if err := overlay.Init(r); err != nil { + fmt.Printf("Failed to initialize overlay driver: %v\n", err) + os.Exit(1) + } + + opt := make(map[string]interface{}) + if len(os.Args) > 1 { + opt[netlabel.OverlayBindInterface] = os.Args[1] + } + if len(os.Args) > 2 { + opt[netlabel.OverlayNeighborIP] = os.Args[2] + } + if len(os.Args) > 3 { + opt[netlabel.KVProvider] = os.Args[3] + } + if len(os.Args) > 4 { + opt[netlabel.KVProviderURL] = os.Args[4] + } + + r.d.Config(opt) + + if err := r.d.CreateNetwork(types.UUID("testnetwork"), + map[string]interface{}{}); err != nil { + fmt.Printf("Failed to create network in the driver: %v\n", err) + os.Exit(1) + } + + ep := &endpoint{} + if err := r.d.CreateEndpoint(types.UUID("testnetwork"), types.UUID("testep"), + ep, map[string]interface{}{}); err != nil { + fmt.Printf("Failed to create endpoint in the driver: %v\n", err) + os.Exit(1) + } + + if err := r.d.Join(types.UUID("testnetwork"), types.UUID("testep"), + "", ep, map[string]interface{}{}); err != nil { + fmt.Printf("Failed to join an endpoint in the driver: %v\n", err) + os.Exit(1) + } + + link, err := netlink.LinkByName(ep.name) + if err != nil { + fmt.Printf("Failed to find the container interface with name %s: %v\n", + ep.name, err) + os.Exit(1) + } + + ipAddr := &netlink.Addr{IPNet: &ep.addr, Label: ""} + if err := netlink.AddrAdd(link, ipAddr); err != nil { + fmt.Printf("Failed to add address to the interface: %v\n", err) + os.Exit(1) + } + + sigCh := make(chan os.Signal, 1) + signal.Notify(sigCh, os.Interrupt, os.Kill) + + for { + select { + case <-sigCh: + r.d.Leave(types.UUID("testnetwork"), types.UUID("testep")) + overlay.Fini(r.d) + os.Exit(0) + } + } +} diff --git a/Godeps/_workspace/src/github.com/docker/libnetwork/cmd/readme_test/readme.go b/Godeps/_workspace/src/github.com/docker/libnetwork/cmd/readme_test/readme.go new file mode 100644 index 000000000..669b517f4 --- /dev/null +++ b/Godeps/_workspace/src/github.com/docker/libnetwork/cmd/readme_test/readme.go @@ -0,0 +1,65 @@ +package main + +import ( + "fmt" + + "github.com/docker/libnetwork" + "github.com/docker/libnetwork/netlabel" + "github.com/docker/libnetwork/options" + "github.com/docker/libnetwork/types" +) + +func main() { + // Create a new controller instance + controller, err := libnetwork.New() + if err != nil { + return + } + + // Select and configure the network driver + networkType := "bridge" + + driverOptions := options.Generic{} + genericOption := make(map[string]interface{}) + genericOption[netlabel.GenericData] = driverOptions + err = controller.ConfigureNetworkDriver(networkType, genericOption) + if err != nil { + return + } + + // Create a network for containers to join. + // NewNetwork accepts Variadic optional arguments that libnetwork and Drivers can make of + network, err := controller.NewNetwork(networkType, "network1") + if err != nil { + return + } + + // For each new container: allocate IP and interfaces. The returned network + // settings will be used for container infos (inspect and such), as well as + // iptables rules for port publishing. This info is contained or accessible + // from the returned endpoint. + ep, err := network.CreateEndpoint("Endpoint1") + if err != nil { + return + } + + // A container can join the endpoint by providing the container ID to the join + // api. + // Join accepts Variadic arguments which will be made use of by libnetwork and Drivers + err = ep.Join("container1", + libnetwork.JoinOptionHostname("test"), + libnetwork.JoinOptionDomainname("docker.io")) + if err != nil { + return + } + + // libnetwork client can check the endpoint's operational data via the Info() API + epInfo, err := ep.DriverInfo() + mapData, ok := epInfo[netlabel.PortMap] + if ok { + portMapping, ok := mapData.([]types.PortBinding) + if ok { + fmt.Printf("Current port mapping for endpoint %s: %v", ep.Name(), portMapping) + } + } +} diff --git a/Godeps/_workspace/src/github.com/docker/libnetwork/cmd/test/libnetwork.toml b/Godeps/_workspace/src/github.com/docker/libnetwork/cmd/test/libnetwork.toml new file mode 100644 index 000000000..4e22516d1 --- /dev/null +++ b/Godeps/_workspace/src/github.com/docker/libnetwork/cmd/test/libnetwork.toml @@ -0,0 +1,12 @@ +title = "LibNetwork Configuration file" + +[daemon] + debug = false +[cluster] + discovery = "token://22aa23948f4f6b31230687689636959e" + Address = "1.1.1.1" +[datastore] + embedded = false +[datastore.client] + provider = "consul" + Address = "localhost:8500" diff --git a/Godeps/_workspace/src/github.com/docker/libnetwork/cmd/test/main.go b/Godeps/_workspace/src/github.com/docker/libnetwork/cmd/test/main.go new file mode 100644 index 000000000..957a0cdab --- /dev/null +++ b/Godeps/_workspace/src/github.com/docker/libnetwork/cmd/test/main.go @@ -0,0 +1,49 @@ +package main + +import ( + "fmt" + "net" + "time" + + log "github.com/Sirupsen/logrus" + + "github.com/docker/libnetwork" + "github.com/docker/libnetwork/options" +) + +func main() { + log.SetLevel(log.DebugLevel) + controller, err := libnetwork.New() + if err != nil { + log.Fatal(err) + } + + netType := "null" + ip, net, _ := net.ParseCIDR("192.168.100.1/24") + net.IP = ip + options := options.Generic{"AddressIPv4": net} + + err = controller.ConfigureNetworkDriver(netType, options) + for i := 0; i < 10; i++ { + netw, err := controller.NewNetwork(netType, fmt.Sprintf("Gordon-%d", i)) + if err != nil { + if _, ok := err.(libnetwork.NetworkNameError); !ok { + log.Fatal(err) + } + } else { + fmt.Println("Network Created Successfully :", netw) + } + netw, _ = controller.NetworkByName(fmt.Sprintf("Gordon-%d", i)) + _, err = netw.CreateEndpoint(fmt.Sprintf("Gordon-Ep-%d", i), nil) + if err != nil { + log.Fatalf("Error creating endpoint 1 %v", err) + } + + _, err = netw.CreateEndpoint(fmt.Sprintf("Gordon-Ep2-%d", i), nil) + if err != nil { + log.Fatalf("Error creating endpoint 2 %v", err) + } + + time.Sleep(2 * time.Second) + } +} diff --git a/Godeps/_workspace/src/github.com/docker/libnetwork/config/config.go b/Godeps/_workspace/src/github.com/docker/libnetwork/config/config.go new file mode 100644 index 000000000..bb93d981b --- /dev/null +++ b/Godeps/_workspace/src/github.com/docker/libnetwork/config/config.go @@ -0,0 +1,116 @@ +package config + +import ( + "strings" + + "github.com/BurntSushi/toml" + log "github.com/Sirupsen/logrus" + "github.com/docker/libnetwork/netlabel" +) + +// Config encapsulates configurations of various Libnetwork components +type Config struct { + Daemon DaemonCfg + Cluster ClusterCfg + Datastore DatastoreCfg +} + +// DaemonCfg represents libnetwork core configuration +type DaemonCfg struct { + Debug bool + DefaultNetwork string + DefaultDriver string + Labels []string +} + +// ClusterCfg represents cluster configuration +type ClusterCfg struct { + Discovery string + Address string + Heartbeat uint64 +} + +// DatastoreCfg represents Datastore configuration. +type DatastoreCfg struct { + Embedded bool + Client DatastoreClientCfg +} + +// DatastoreClientCfg represents Datastore Client-only mode configuration +type DatastoreClientCfg struct { + Provider string + Address string +} + +// ParseConfig parses the libnetwork configuration file +func ParseConfig(tomlCfgFile string) (*Config, error) { + var cfg Config + if _, err := toml.DecodeFile(tomlCfgFile, &cfg); err != nil { + return nil, err + } + return &cfg, nil +} + +// Option is a option setter function type used to pass varios configurations +// to the controller +type Option func(c *Config) + +// OptionDefaultNetwork function returns an option setter for a default network +func OptionDefaultNetwork(dn string) Option { + return func(c *Config) { + log.Infof("Option DefaultNetwork: %s", dn) + c.Daemon.DefaultNetwork = strings.TrimSpace(dn) + } +} + +// OptionDefaultDriver function returns an option setter for default driver +func OptionDefaultDriver(dd string) Option { + return func(c *Config) { + log.Infof("Option DefaultDriver: %s", dd) + c.Daemon.DefaultDriver = strings.TrimSpace(dd) + } +} + +// OptionLabels function returns an option setter for labels +func OptionLabels(labels []string) Option { + return func(c *Config) { + for _, label := range labels { + if strings.HasPrefix(label, netlabel.Prefix) { + c.Daemon.Labels = append(c.Daemon.Labels, label) + } + } + } +} + +// OptionKVProvider function returns an option setter for kvstore provider +func OptionKVProvider(provider string) Option { + return func(c *Config) { + log.Infof("Option OptionKVProvider: %s", provider) + c.Datastore.Client.Provider = strings.TrimSpace(provider) + } +} + +// OptionKVProviderURL function returns an option setter for kvstore url +func OptionKVProviderURL(url string) Option { + return func(c *Config) { + log.Infof("Option OptionKVProviderURL: %s", url) + c.Datastore.Client.Address = strings.TrimSpace(url) + } +} + +// ProcessOptions processes options and stores it in config +func (c *Config) ProcessOptions(options ...Option) { + for _, opt := range options { + if opt != nil { + opt(c) + } + } +} + +// IsValidName validates configuration objects supported by libnetwork +func IsValidName(name string) bool { + if name == "" || strings.Contains(name, ".") { + return false + } + return true +} diff --git a/Godeps/_workspace/src/github.com/docker/libnetwork/config/config_test.go b/Godeps/_workspace/src/github.com/docker/libnetwork/config/config_test.go new file mode 100644 index 000000000..cc8a911c9 --- /dev/null +++ b/Godeps/_workspace/src/github.com/docker/libnetwork/config/config_test.go @@ -0,0 +1,55 @@ +package config + +import ( + "strings" + "testing" + + "github.com/docker/libnetwork/netlabel" + _ "github.com/docker/libnetwork/netutils" +) + +func TestInvalidConfig(t *testing.T) { + _, err := ParseConfig("invalid.toml") + if err == nil { + t.Fatal("Invalid Configuration file must fail") + } +} + +func TestConfig(t *testing.T) { + _, err := ParseConfig("libnetwork.toml") + if err != nil { + t.Fatal("Error parsing a valid configuration file :", err) + } +} + +func TestOptionsLabels(t *testing.T) { + c := &Config{} + l := []string{ + "com.docker.network.key1=value1", + "com.docker.storage.key1=value1", + "com.docker.network.driver.key1=value1", + "com.docker.network.driver.key2=value2", + } + f := OptionLabels(l) + f(c) + if len(c.Daemon.Labels) != 3 { + t.Fatalf("Expecting 3 labels, seen %d", len(c.Daemon.Labels)) + } + for _, l := range c.Daemon.Labels { + if !strings.HasPrefix(l, netlabel.Prefix) { + t.Fatalf("config must accept only libnetwork labels. Not : %s", l) + } + } +} + +func TestValidName(t *testing.T) { + if !IsValidName("test") { + t.Fatal("Name validation fails for a name that must be accepted") + } + if IsValidName("") { + t.Fatal("Name validation succeeds for a case when it is expected to fail") + } + if IsValidName("name.with.dots") { + t.Fatal("Name validation succeeds for a case when it is expected to fail") + } +} diff --git a/Godeps/_workspace/src/github.com/docker/libnetwork/config/libnetwork.toml b/Godeps/_workspace/src/github.com/docker/libnetwork/config/libnetwork.toml new file mode 100644 index 000000000..93a2ff475 --- /dev/null +++ b/Godeps/_workspace/src/github.com/docker/libnetwork/config/libnetwork.toml @@ -0,0 +1,12 @@ +title = "LibNetwork Configuration file" + +[daemon] + debug = false +[cluster] + discovery = "token://swarm-discovery-token" + Address = "Cluster-wide reachable Host IP" +[datastore] + embedded = false +[datastore.client] + provider = "consul" + Address = "localhost:8500" diff --git a/Godeps/_workspace/src/github.com/docker/libnetwork/controller.go b/Godeps/_workspace/src/github.com/docker/libnetwork/controller.go new file mode 100644 index 000000000..02a9f7eb2 --- /dev/null +++ b/Godeps/_workspace/src/github.com/docker/libnetwork/controller.go @@ -0,0 +1,399 @@ +/* +Package libnetwork provides the basic functionality and extension points to +create network namespaces and allocate interfaces for containers to use. + + // Create a new controller instance + controller, _err := libnetwork.New(nil) + + // Select and configure the network driver + networkType := "bridge" + + driverOptions := options.Generic{} + genericOption := make(map[string]interface{}) + genericOption[netlabel.GenericData] = driverOptions + err := controller.ConfigureNetworkDriver(networkType, genericOption) + if err != nil { + return + } + + // Create a network for containers to join. + // NewNetwork accepts Variadic optional arguments that libnetwork and Drivers can make of + network, err := controller.NewNetwork(networkType, "network1") + if err != nil { + return + } + + // For each new container: allocate IP and interfaces. The returned network + // settings will be used for container infos (inspect and such), as well as + // iptables rules for port publishing. This info is contained or accessible + // from the returned endpoint. + ep, err := network.CreateEndpoint("Endpoint1") + if err != nil { + return + } + + // A container can join the endpoint by providing the container ID to the join + // api. + // Join accepts Variadic arguments which will be made use of by libnetwork and Drivers + err = ep.Join("container1", + libnetwork.JoinOptionHostname("test"), + libnetwork.JoinOptionDomainname("docker.io")) + if err != nil { + return + } +*/ +package libnetwork + +import ( + "fmt" + "net" + "strings" + "sync" + + log "github.com/Sirupsen/logrus" + "github.com/docker/docker/pkg/plugins" + "github.com/docker/docker/pkg/stringid" + "github.com/docker/libnetwork/config" + "github.com/docker/libnetwork/datastore" + "github.com/docker/libnetwork/driverapi" + "github.com/docker/libnetwork/hostdiscovery" + "github.com/docker/libnetwork/netlabel" + "github.com/docker/libnetwork/sandbox" + "github.com/docker/libnetwork/types" +) + +// NetworkController provides the interface for controller instance which manages +// networks. +type NetworkController interface { + // ConfigureNetworkDriver applies the passed options to the driver instance for the specified network type + ConfigureNetworkDriver(networkType string, options map[string]interface{}) error + + // Config method returns the bootup configuration for the controller + Config() config.Config + + // Create a new network. The options parameter carries network specific options. + // Labels support will be added in the near future. + NewNetwork(networkType, name string, options ...NetworkOption) (Network, error) + + // Networks returns the list of Network(s) managed by this controller. + Networks() []Network + + // WalkNetworks uses the provided function to walk the Network(s) managed by this controller. + WalkNetworks(walker NetworkWalker) + + // NetworkByName returns the Network which has the passed name. If not found, the error ErrNoSuchNetwork is returned. + NetworkByName(name string) (Network, error) + + // NetworkByID returns the Network which has the passed id. If not found, the error ErrNoSuchNetwork is returned. + NetworkByID(id string) (Network, error) + + // LeaveAll accepts a container id and attempts to leave all endpoints that the container has joined + LeaveAll(id string) error + + // GC triggers immediate garbage collection of resources which are garbage collected. + GC() +} + +// NetworkWalker is a client provided function which will be used to walk the Networks. +// When the function returns true, the walk will stop. +type NetworkWalker func(nw Network) bool + +type driverData struct { + driver driverapi.Driver + capability driverapi.Capability +} + +type driverTable map[string]*driverData +type networkTable map[types.UUID]*network +type endpointTable map[types.UUID]*endpoint +type sandboxTable map[string]*sandboxData + +type controller struct { + networks networkTable + drivers driverTable + sandboxes sandboxTable + cfg *config.Config + store datastore.DataStore + sync.Mutex +} + +// New creates a new instance of network controller. +func New(cfgOptions ...config.Option) (NetworkController, error) { + var cfg *config.Config + if len(cfgOptions) > 0 { + cfg = &config.Config{} + cfg.ProcessOptions(cfgOptions...) + } + c := &controller{ + cfg: cfg, + networks: networkTable{}, + sandboxes: sandboxTable{}, + drivers: driverTable{}} + if err := initDrivers(c); err != nil { + return nil, err + } + + if cfg != nil { + if err := c.initDataStore(); err != nil { + // Failing to initalize datastore is a bad situation to be in. + // But it cannot fail creating the Controller + log.Debugf("Failed to Initialize Datastore due to %v. Operating in non-clustered mode", err) + } + if err := c.initDiscovery(); err != nil { + // Failing to initalize discovery is a bad situation to be in. + // But it cannot fail creating the Controller + log.Debugf("Failed to Initialize Discovery : %v", err) + } + } + + return c, nil +} + +func (c *controller) validateHostDiscoveryConfig() bool { + if c.cfg == nil || c.cfg.Cluster.Discovery == "" || c.cfg.Cluster.Address == "" { + return false + } + return true +} + +func (c *controller) initDiscovery() error { + if c.cfg == nil { + return fmt.Errorf("discovery initialization requires a valid configuration") + } + + hostDiscovery := hostdiscovery.NewHostDiscovery() + return hostDiscovery.StartDiscovery(&c.cfg.Cluster, c.hostJoinCallback, c.hostLeaveCallback) +} + +func (c *controller) hostJoinCallback(hosts []net.IP) { +} + +func (c *controller) hostLeaveCallback(hosts []net.IP) { +} + +func (c *controller) Config() config.Config { + c.Lock() + defer c.Unlock() + if c.cfg == nil { + return config.Config{} + } + return *c.cfg +} + +func (c *controller) ConfigureNetworkDriver(networkType string, options map[string]interface{}) error { + c.Lock() + dd, ok := c.drivers[networkType] + c.Unlock() + if !ok { + return NetworkTypeError(networkType) + } + return dd.driver.Config(options) +} + +func (c *controller) RegisterDriver(networkType string, driver driverapi.Driver, capability driverapi.Capability) error { + c.Lock() + if !config.IsValidName(networkType) { + c.Unlock() + return ErrInvalidName(networkType) + } + if _, ok := c.drivers[networkType]; ok { + c.Unlock() + return driverapi.ErrActiveRegistration(networkType) + } + c.drivers[networkType] = &driverData{driver, capability} + + if c.cfg == nil { + c.Unlock() + return nil + } + + opt := make(map[string]interface{}) + for _, label := range c.cfg.Daemon.Labels { + if strings.HasPrefix(label, netlabel.DriverPrefix+"."+networkType) { + opt[netlabel.Key(label)] = netlabel.Value(label) + } + } + + if capability.Scope == driverapi.GlobalScope && c.validateDatastoreConfig() { + opt[netlabel.KVProvider] = c.cfg.Datastore.Client.Provider + opt[netlabel.KVProviderURL] = c.cfg.Datastore.Client.Address + } + + c.Unlock() + + if len(opt) != 0 { + if err := driver.Config(opt); err != nil { + return err + } + } + + return nil +} + +// NewNetwork creates a new network of the specified network type. The options +// are network specific and modeled in a generic way. +func (c *controller) NewNetwork(networkType, name string, options ...NetworkOption) (Network, error) { + if !config.IsValidName(name) { + return nil, ErrInvalidName(name) + } + // Check if a network already exists with the specified network name + c.Lock() + for _, n := range c.networks { + if n.name == name { + c.Unlock() + return nil, NetworkNameError(name) + } + } + c.Unlock() + + // Construct the network object + network := &network{ + name: name, + networkType: networkType, + id: types.UUID(stringid.GenerateRandomID()), + ctrlr: c, + endpoints: endpointTable{}, + } + + network.processOptions(options...) + + if err := c.addNetwork(network); err != nil { + return nil, err + } + + if err := c.updateNetworkToStore(network); err != nil { + log.Warnf("couldnt create network %s: %v", network.name, err) + if e := network.Delete(); e != nil { + log.Warnf("couldnt cleanup network %s: %v", network.name, err) + } + return nil, err + } + + return network, nil +} + +func (c *controller) addNetwork(n *network) error { + + c.Lock() + // Check if a driver for the specified network type is available + dd, ok := c.drivers[n.networkType] + c.Unlock() + + if !ok { + var err error + dd, err = c.loadDriver(n.networkType) + if err != nil { + return err + } + } + + n.Lock() + n.svcRecords = svcMap{} + n.driver = dd.driver + d := n.driver + n.Unlock() + + // Create the network + if err := d.CreateNetwork(n.id, n.generic); err != nil { + return err + } + if err := n.watchEndpoints(); err != nil { + return err + } + c.Lock() + c.networks[n.id] = n + c.Unlock() + + return nil +} + +func (c *controller) Networks() []Network { + c.Lock() + defer c.Unlock() + + list := make([]Network, 0, len(c.networks)) + for _, n := range c.networks { + list = append(list, n) + } + + return list +} + +func (c *controller) WalkNetworks(walker NetworkWalker) { + for _, n := range c.Networks() { + if walker(n) { + return + } + } +} + +func (c *controller) NetworkByName(name string) (Network, error) { + if name == "" { + return nil, ErrInvalidName(name) + } + var n Network + + s := func(current Network) bool { + if current.Name() == name { + n = current + return true + } + return false + } + + c.WalkNetworks(s) + + if n == nil { + return nil, ErrNoSuchNetwork(name) + } + + return n, nil +} + +func (c *controller) NetworkByID(id string) (Network, error) { + if id == "" { + return nil, ErrInvalidID(id) + } + c.Lock() + defer c.Unlock() + if n, ok := c.networks[types.UUID(id)]; ok { + return n, nil + } + return nil, ErrNoSuchNetwork(id) +} + +func (c *controller) loadDriver(networkType string) (*driverData, error) { + // Plugins pkg performs lazy loading of plugins that acts as remote drivers. + // As per the design, this Get call will result in remote driver discovery if there is a corresponding plugin available. + _, err := plugins.Get(networkType, driverapi.NetworkPluginEndpointType) + if err != nil { + if err == plugins.ErrNotFound { + return nil, types.NotFoundErrorf(err.Error()) + } + return nil, err + } + c.Lock() + defer c.Unlock() + dd, ok := c.drivers[networkType] + if !ok { + return nil, ErrInvalidNetworkDriver(networkType) + } + return dd, nil +} + +func (c *controller) isDriverGlobalScoped(networkType string) (bool, error) { + c.Lock() + dd, ok := c.drivers[networkType] + c.Unlock() + if !ok { + return false, types.NotFoundErrorf("driver not found for %s", networkType) + } + if dd.capability.Scope == driverapi.GlobalScope { + return true, nil + } + return false, nil +} + +func (c *controller) GC() { + sandbox.GC() +} diff --git a/Godeps/_workspace/src/github.com/docker/libnetwork/datastore/datastore.go b/Godeps/_workspace/src/github.com/docker/libnetwork/datastore/datastore.go new file mode 100644 index 000000000..8a195aa90 --- /dev/null +++ b/Godeps/_workspace/src/github.com/docker/libnetwork/datastore/datastore.go @@ -0,0 +1,194 @@ +package datastore + +import ( + "reflect" + "strings" + + "github.com/docker/libkv" + "github.com/docker/libkv/store" + "github.com/docker/libnetwork/config" + "github.com/docker/libnetwork/types" +) + +//DataStore exported +type DataStore interface { + // GetObject gets data from datastore and unmarshals to the specified object + GetObject(key string, o KV) error + // PutObject adds a new Record based on an object into the datastore + PutObject(kvObject KV) error + // PutObjectAtomic provides an atomic add and update operation for a Record + PutObjectAtomic(kvObject KV) error + // DeleteObject deletes a record + DeleteObject(kvObject KV) error + // DeleteObjectAtomic performs an atomic delete operation + DeleteObjectAtomic(kvObject KV) error + // DeleteTree deletes a record + DeleteTree(kvObject KV) error + // KVStore returns access to the KV Store + KVStore() store.Store +} + +// ErrKeyModified is raised for an atomic update when the update is working on a stale state +var ( + ErrKeyModified = store.ErrKeyModified + ErrKeyNotFound = store.ErrKeyNotFound +) + +type datastore struct { + store store.Store +} + +//KV Key Value interface used by objects to be part of the DataStore +type KV interface { + // Key method lets an object to provide the Key to be used in KV Store + Key() []string + // KeyPrefix method lets an object to return immediate parent key that can be used for tree walk + KeyPrefix() []string + // Value method lets an object to marshal its content to be stored in the KV store + Value() []byte + // SetValue is used by the datastore to set the object's value when loaded from the data store. + SetValue([]byte) error + // Index method returns the latest DB Index as seen by the object + Index() uint64 + // SetIndex method allows the datastore to store the latest DB Index into the object + SetIndex(uint64) + // True if the object exists in the datastore, false if it hasn't been stored yet. + // When SetIndex() is called, the object has been stored. + Exists() bool +} + +const ( + // NetworkKeyPrefix is the prefix for network key in the kv store + NetworkKeyPrefix = "network" + // EndpointKeyPrefix is the prefix for endpoint key in the kv store + EndpointKeyPrefix = "endpoint" +) + +var rootChain = []string{"docker", "libnetwork"} + +//Key provides convenient method to create a Key +func Key(key ...string) string { + keychain := append(rootChain, key...) + str := strings.Join(keychain, "/") + return str + "/" +} + +//ParseKey provides convenient method to unpack the key to complement the Key function +func ParseKey(key string) ([]string, error) { + chain := strings.Split(strings.Trim(key, "/"), "/") + + // The key must atleast be equal to the rootChain in order to be considered as valid + if len(chain) <= len(rootChain) || !reflect.DeepEqual(chain[0:len(rootChain)], rootChain) { + return nil, types.BadRequestErrorf("invalid Key : %s", key) + } + return chain[len(rootChain):], nil +} + +// newClient used to connect to KV Store +func newClient(kv string, addrs string) (DataStore, error) { + store, err := libkv.NewStore(store.Backend(kv), []string{addrs}, &store.Config{}) + if err != nil { + return nil, err + } + ds := &datastore{store: store} + return ds, nil +} + +// NewDataStore creates a new instance of LibKV data store +func NewDataStore(cfg *config.DatastoreCfg) (DataStore, error) { + if cfg == nil { + return nil, types.BadRequestErrorf("invalid configuration passed to datastore") + } + // TODO : cfg.Embedded case + return newClient(cfg.Client.Provider, cfg.Client.Address) +} + +// NewCustomDataStore can be used by clients to plugin cusom datatore that adhers to store.Store +func NewCustomDataStore(customStore store.Store) DataStore { + return &datastore{store: customStore} +} + +func (ds *datastore) KVStore() store.Store { + return ds.store +} + +// PutObjectAtomic adds a new Record based on an object into the datastore +func (ds *datastore) PutObjectAtomic(kvObject KV) error { + if kvObject == nil { + return types.BadRequestErrorf("invalid KV Object : nil") + } + kvObjValue := kvObject.Value() + + if kvObjValue == nil { + return types.BadRequestErrorf("invalid KV Object with a nil Value for key %s", Key(kvObject.Key()...)) + } + + var previous *store.KVPair + if kvObject.Exists() { + previous = &store.KVPair{Key: Key(kvObject.Key()...), LastIndex: kvObject.Index()} + } else { + previous = nil + } + _, pair, err := ds.store.AtomicPut(Key(kvObject.Key()...), kvObjValue, previous, nil) + if err != nil { + return err + } + + kvObject.SetIndex(pair.LastIndex) + return nil +} + +// PutObject adds a new Record based on an object into the datastore +func (ds *datastore) PutObject(kvObject KV) error { + if kvObject == nil { + return types.BadRequestErrorf("invalid KV Object : nil") + } + return ds.putObjectWithKey(kvObject, kvObject.Key()...) +} + +func (ds *datastore) putObjectWithKey(kvObject KV, key ...string) error { + kvObjValue := kvObject.Value() + + if kvObjValue == nil { + return types.BadRequestErrorf("invalid KV Object with a nil Value for key %s", Key(kvObject.Key()...)) + } + return ds.store.Put(Key(key...), kvObjValue, nil) +} + +// GetObject returns a record matching the key +func (ds *datastore) GetObject(key string, o KV) error { + kvPair, err := ds.store.Get(key) + if err != nil { + return err + } + err = o.SetValue(kvPair.Value) + if err != nil { + return err + } + + // Make sure the object has a correct view of the DB index in case we need to modify it + // and update the DB. + o.SetIndex(kvPair.LastIndex) + return nil +} + +// DeleteObject unconditionally deletes a record from the store +func (ds *datastore) DeleteObject(kvObject KV) error { + return ds.store.Delete(Key(kvObject.Key()...)) +} + +// DeleteObjectAtomic performs atomic delete on a record +func (ds *datastore) DeleteObjectAtomic(kvObject KV) error { + if kvObject == nil { + return types.BadRequestErrorf("invalid KV Object : nil") + } + + previous := &store.KVPair{Key: Key(kvObject.Key()...), LastIndex: kvObject.Index()} + _, err := ds.store.AtomicDelete(Key(kvObject.Key()...), previous) + return err +} + +// DeleteTree unconditionally deletes a record from the store +func (ds *datastore) DeleteTree(kvObject KV) error { + return ds.store.DeleteTree(Key(kvObject.KeyPrefix()...)) +} diff --git a/Godeps/_workspace/src/github.com/docker/libnetwork/datastore/datastore_test.go b/Godeps/_workspace/src/github.com/docker/libnetwork/datastore/datastore_test.go new file mode 100644 index 000000000..397842db1 --- /dev/null +++ b/Godeps/_workspace/src/github.com/docker/libnetwork/datastore/datastore_test.go @@ -0,0 +1,241 @@ +package datastore + +import ( + "encoding/json" + "reflect" + "testing" + + "github.com/docker/libnetwork/config" + _ "github.com/docker/libnetwork/netutils" + "github.com/docker/libnetwork/options" + "github.com/stretchr/testify/assert" +) + +var dummyKey = "dummy" + +// NewCustomDataStore can be used by other Tests in order to use custom datastore +func NewTestDataStore() DataStore { + return &datastore{store: NewMockStore()} +} + +func TestKey(t *testing.T) { + eKey := []string{"hello", "world"} + sKey := Key(eKey...) + if sKey != "docker/libnetwork/hello/world/" { + t.Fatalf("unexpected key : %s", sKey) + } +} + +func TestParseKey(t *testing.T) { + keySlice, err := ParseKey("/docker/libnetwork/hello/world/") + if err != nil { + t.Fatal(err) + } + eKey := []string{"hello", "world"} + if len(keySlice) < 2 || !reflect.DeepEqual(eKey, keySlice) { + t.Fatalf("unexpected unkey : %s", keySlice) + } +} + +func TestInvalidDataStore(t *testing.T) { + config := &config.DatastoreCfg{} + config.Embedded = false + config.Client.Provider = "invalid" + config.Client.Address = "localhost:8500" + _, err := NewDataStore(config) + if err == nil { + t.Fatal("Invalid Datastore connection configuration must result in a failure") + } +} + +func TestKVObjectFlatKey(t *testing.T) { + store := NewTestDataStore() + expected := dummyKVObject("1000", true) + err := store.PutObject(expected) + if err != nil { + t.Fatal(err) + } + keychain := []string{dummyKey, "1000"} + data, err := store.KVStore().Get(Key(keychain...)) + if err != nil { + t.Fatal(err) + } + var n dummyObject + json.Unmarshal(data.Value, &n) + if n.Name != expected.Name { + t.Fatalf("Dummy object doesn't match the expected object") + } +} + +func TestAtomicKVObjectFlatKey(t *testing.T) { + store := NewTestDataStore() + expected := dummyKVObject("1111", true) + assert.False(t, expected.Exists()) + err := store.PutObjectAtomic(expected) + if err != nil { + t.Fatal(err) + } + assert.True(t, expected.Exists()) + + // PutObjectAtomic automatically sets the Index again. Hence the following must pass. + + err = store.PutObjectAtomic(expected) + if err != nil { + t.Fatal("Atomic update should succeed.") + } + + // Get the latest index and try PutObjectAtomic again for the same Key + // This must succeed as well + data, err := store.KVStore().Get(Key(expected.Key()...)) + if err != nil { + t.Fatal(err) + } + n := dummyObject{} + json.Unmarshal(data.Value, &n) + n.ID = "1111" + n.SetIndex(data.LastIndex) + n.ReturnValue = true + err = store.PutObjectAtomic(&n) + if err != nil { + t.Fatal(err) + } + + // Get the Object using GetObject, then set again. + newObj := dummyObject{} + err = store.GetObject(Key(expected.Key()...), &newObj) + assert.True(t, newObj.Exists()) + err = store.PutObjectAtomic(&n) + if err != nil { + t.Fatal(err) + } + +} + +// dummy data used to test the datastore +type dummyObject struct { + Name string `kv:"leaf"` + NetworkType string `kv:"leaf"` + EnableIPv6 bool `kv:"leaf"` + Rec *recStruct `kv:"recursive"` + Dict map[string]*recStruct `kv:"iterative"` + Generic options.Generic `kv:"iterative"` + ID string + DBIndex uint64 + DBExists bool + ReturnValue bool +} + +func (n *dummyObject) Key() []string { + return []string{dummyKey, n.ID} +} + +func (n *dummyObject) KeyPrefix() []string { + return []string{dummyKey} +} + +func (n *dummyObject) Value() []byte { + if !n.ReturnValue { + return nil + } + + b, err := json.Marshal(n) + if err != nil { + return nil + } + return b +} + +func (n *dummyObject) SetValue(value []byte) error { + return json.Unmarshal(value, n) +} + +func (n *dummyObject) Index() uint64 { + return n.DBIndex +} + +func (n *dummyObject) SetIndex(index uint64) { + n.DBIndex = index + n.DBExists = true +} + +func (n *dummyObject) Exists() bool { + return n.DBExists +} + +func (n *dummyObject) MarshalJSON() ([]byte, error) { + netMap := make(map[string]interface{}) + netMap["name"] = n.Name + netMap["networkType"] = n.NetworkType + netMap["enableIPv6"] = n.EnableIPv6 + netMap["generic"] = n.Generic + return json.Marshal(netMap) +} + +func (n *dummyObject) UnmarshalJSON(b []byte) (err error) { + var netMap map[string]interface{} + if err := json.Unmarshal(b, &netMap); err != nil { + return err + } + n.Name = netMap["name"].(string) + n.NetworkType = netMap["networkType"].(string) + n.EnableIPv6 = netMap["enableIPv6"].(bool) + n.Generic = netMap["generic"].(map[string]interface{}) + return nil +} + +// dummy structure to test "recursive" cases +type recStruct struct { + Name string `kv:"leaf"` + Field1 int `kv:"leaf"` + Dict map[string]string `kv:"iterative"` + DBIndex uint64 + DBExists bool +} + +func (r *recStruct) Key() []string { + return []string{"recStruct"} +} +func (r *recStruct) Value() []byte { + b, err := json.Marshal(r) + if err != nil { + return nil + } + return b +} + +func (r *recStruct) SetValue(value []byte) error { + return json.Unmarshal(value, r) +} + +func (r *recStruct) Index() uint64 { + return r.DBIndex +} + +func (r *recStruct) SetIndex(index uint64) { + r.DBIndex = index + r.DBExists = true +} + +func (r *recStruct) Exists() bool { + return r.DBExists +} + +func dummyKVObject(id string, retValue bool) *dummyObject { + cDict := make(map[string]string) + cDict["foo"] = "bar" + cDict["hello"] = "world" + n := dummyObject{ + Name: "testNw", + NetworkType: "bridge", + EnableIPv6: true, + Rec: &recStruct{"gen", 5, cDict, 0, false}, + ID: id, + DBIndex: 0, + ReturnValue: retValue, + DBExists: false} + generic := make(map[string]interface{}) + generic["label1"] = &recStruct{"value1", 1, cDict, 0, false} + generic["label2"] = "subnet=10.1.1.0/16" + n.Generic = generic + return &n +} diff --git a/Godeps/_workspace/src/github.com/docker/libnetwork/datastore/mock_store.go b/Godeps/_workspace/src/github.com/docker/libnetwork/datastore/mock_store.go new file mode 100644 index 000000000..0817339b6 --- /dev/null +++ b/Godeps/_workspace/src/github.com/docker/libnetwork/datastore/mock_store.go @@ -0,0 +1,129 @@ +package datastore + +import ( + "errors" + + "github.com/docker/libkv/store" + "github.com/docker/libnetwork/types" +) + +var ( + // ErrNotImplmented exported + ErrNotImplmented = errors.New("Functionality not implemented") +) + +// MockData exported +type MockData struct { + Data []byte + Index uint64 +} + +// MockStore exported +type MockStore struct { + db map[string]*MockData +} + +// NewMockStore creates a Map backed Datastore that is useful for mocking +func NewMockStore() *MockStore { + db := make(map[string]*MockData) + return &MockStore{db} +} + +// Get the value at "key", returns the last modified index +// to use in conjunction to CAS calls +func (s *MockStore) Get(key string) (*store.KVPair, error) { + mData := s.db[key] + if mData == nil { + return nil, nil + } + return &store.KVPair{Value: mData.Data, LastIndex: mData.Index}, nil + +} + +// Put a value at "key" +func (s *MockStore) Put(key string, value []byte, options *store.WriteOptions) error { + mData := s.db[key] + if mData == nil { + mData = &MockData{value, 0} + } + mData.Index = mData.Index + 1 + s.db[key] = mData + return nil +} + +// Delete a value at "key" +func (s *MockStore) Delete(key string) error { + delete(s.db, key) + return nil +} + +// Exists checks that the key exists inside the store +func (s *MockStore) Exists(key string) (bool, error) { + _, ok := s.db[key] + return ok, nil +} + +// List gets a range of values at "directory" +func (s *MockStore) List(prefix string) ([]*store.KVPair, error) { + return nil, ErrNotImplmented +} + +// DeleteTree deletes a range of values at "directory" +func (s *MockStore) DeleteTree(prefix string) error { + delete(s.db, prefix) + return nil +} + +// Watch a single key for modifications +func (s *MockStore) Watch(key string, stopCh <-chan struct{}) (<-chan *store.KVPair, error) { + return nil, ErrNotImplmented +} + +// WatchTree triggers a watch on a range of values at "directory" +func (s *MockStore) WatchTree(prefix string, stopCh <-chan struct{}) (<-chan []*store.KVPair, error) { + return nil, ErrNotImplmented +} + +// NewLock exposed +func (s *MockStore) NewLock(key string, options *store.LockOptions) (store.Locker, error) { + return nil, ErrNotImplmented +} + +// AtomicPut put a value at "key" if the key has not been +// modified in the meantime, throws an error if this is the case +func (s *MockStore) AtomicPut(key string, newValue []byte, previous *store.KVPair, options *store.WriteOptions) (bool, *store.KVPair, error) { + mData := s.db[key] + + if previous == nil { + if mData != nil { + return false, nil, types.BadRequestErrorf("atomic put failed because key exists") + } // Else OK. + } else { + if mData == nil { + return false, nil, types.BadRequestErrorf("atomic put failed because key exists") + } + if mData != nil && mData.Index != previous.LastIndex { + return false, nil, types.BadRequestErrorf("atomic put failed due to mismatched Index") + } // Else OK. + } + err := s.Put(key, newValue, nil) + if err != nil { + return false, nil, err + } + return true, &store.KVPair{Key: key, Value: newValue, LastIndex: s.db[key].Index}, nil +} + +// AtomicDelete deletes a value at "key" if the key has not +// been modified in the meantime, throws an error if this is the case +func (s *MockStore) AtomicDelete(key string, previous *store.KVPair) (bool, error) { + mData := s.db[key] + if mData != nil && mData.Index != previous.LastIndex { + return false, types.BadRequestErrorf("atomic delete failed due to mismatched Index") + } + return true, s.Delete(key) +} + +// Close closes the client connection +func (s *MockStore) Close() { + return +} diff --git a/Godeps/_workspace/src/github.com/docker/libnetwork/docs/Vagrantfile b/Godeps/_workspace/src/github.com/docker/libnetwork/docs/Vagrantfile new file mode 100644 index 000000000..55d665f19 --- /dev/null +++ b/Godeps/_workspace/src/github.com/docker/libnetwork/docs/Vagrantfile @@ -0,0 +1,58 @@ +# -*- mode: ruby -*- +# vi: set ft=ruby : + +# Vagrantfile API/syntax version. Don't touch unless you know what you're doing! +VAGRANTFILE_API_VERSION = "2" + +$consul=<