Skip to content

Commit

Permalink
Merge pull request #1213 from garden-io/fix-hot-reload
Browse files Browse the repository at this point in the history 
Fix hot reload issues
  • Loading branch information
@eysi09
eysi09 authored Sep 20, 2019
2 parents 373beeb + b71a8ab commit 06ed6fd
Show file tree
Hide file tree
Showing 32 changed files with 336 additions and 251 deletions.
4 changes: 2 additions & 2 deletions garden-service/src/actions.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,7 @@ import { HotReloadServiceParams, HotReloadServiceResult } from "./types/plugin/s
import { RunServiceParams } from "./types/plugin/service/runService"
import { GetTaskResultParams } from "./types/plugin/task/getTaskResult"
import { RunTaskParams, RunTaskResult } from "./types/plugin/task/runTask"
import { ServiceStatus, ServiceStatusMap } from "./types/service"
import { ServiceStatus, ServiceStatusMap, ServiceState } from "./types/service"
import { Omit } from "./util/util"
import { DebugInfoMap } from "./types/plugin/provider/getDebugInfo"
import { PrepareEnvironmentParams, PrepareEnvironmentResult } from "./types/plugin/provider/prepareEnvironment"
Expand Down Expand Up @@ -784,5 +784,5 @@ const dummyPublishHandler = async ({ module }) => {
const dummyDeleteServiceHandler = async ({ module, log }: DeleteServiceParams) => {
const msg = `No delete service handler available for module type ${module.type}`
log.setError(msg)
return {}
return { state: "missing" as ServiceState, detail: {} }
}
4 changes: 2 additions & 2 deletions garden-service/src/plugins/google/google-app-engine.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,7 @@ export const gardenPlugin = (): GardenPlugin => ({
// const services = await this.gcloud(project).json(["app", "services", "list"])
// const instances: any[] = await this.gcloud(project).json(["app", "instances", "list"])

return {}
return { state: "unknown", detail: {} }
},

async deployService({ ctx, service, runtimeContext, log }: DeployServiceParams<ContainerModule>) {
Expand Down Expand Up @@ -104,7 +104,7 @@ export const gardenPlugin = (): GardenPlugin => ({

log.info({ section: service.name, msg: `App deployed` })

return {}
return { state: "ready", detail: {} }
},
},
},
Expand Down
2 changes: 1 addition & 1 deletion garden-service/src/plugins/google/google-cloud-functions.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -144,7 +144,7 @@ export async function getServiceStatus(

if (!status) {
// not deployed yet
return {}
return { state: "missing", detail: {} }
}

// TODO: map states properly
Expand Down
6 changes: 3 additions & 3 deletions garden-service/src/plugins/kubernetes/commands/cleanup-cluster-registry.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ import { systemNamespace } from "../system"
import { PluginError } from "../../../exceptions"
import { apply, kubectl } from "../kubectl"
import { waitForResources } from "../status/status"
import { execInDeployment } from "../container/run"
import { execInWorkload } from "../container/run"
import { dedent, deline } from "../../../util/string"
import { execInBuilder, getBuilderPodName, BuilderExecParams, buildSyncDeploymentName } from "../container/build"
import { getPods } from "../util"
Expand Down Expand Up @@ -226,11 +226,11 @@ async function runRegistryGarbageCollection(ctx: KubernetesPluginContext, api: K

// Run garbage collection
log.info("Running garbage collection...")
await execInDeployment({
await execInWorkload({
provider,
log,
namespace: systemNamespace,
deploymentName: CLUSTER_REGISTRY_DEPLOYMENT_NAME,
workload: modifiedDeployment,
command: ["/bin/registry", "garbage-collect", "/etc/docker/registry/config.yml"],
interactive: false,
})
Expand Down
165 changes: 53 additions & 112 deletions garden-service/src/plugins/kubernetes/container/deployment.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,35 +7,36 @@
*/

import chalk from "chalk"
import Bluebird from "bluebird"
import { V1Container } from "@kubernetes/client-node"
import { Service, ServiceStatus } from "../../../types/service"
import { Service } from "../../../types/service"
import { extend, find, keyBy, merge, set } from "lodash"
import { ContainerModule, ContainerService } from "../../container/config"
import { createIngressResources } from "./ingress"
import { createServiceResources } from "./service"
import { waitForResources, compareDeployedObjects } from "../status/status"
import { apply, deleteObjectsByLabel } from "../kubectl"
import { apply, deleteObjectsBySelector } from "../kubectl"
import { getAppNamespace } from "../namespace"
import { PluginContext } from "../../../plugin-context"
import { KubeApi } from "../api"
import { KubernetesProvider, KubernetesPluginContext } from "../config"
import { configureHotReload } from "../hot-reload"
import { KubernetesResource, KubernetesServerResource } from "../types"
import { KubernetesWorkload } from "../types"
import { ConfigurationError } from "../../../exceptions"
import { getContainerServiceStatus } from "./status"
import { getContainerServiceStatus, ContainerServiceStatus } from "./status"
import { containerHelpers } from "../../container/helpers"
import { LogEntry } from "../../../logger/log-entry"
import { DeployServiceParams } from "../../../types/plugin/service/deployService"
import { DeleteServiceParams } from "../../../types/plugin/service/deleteService"
import { millicpuToString, kilobytesToString, prepareEnvVars } from "../util"
import { millicpuToString, kilobytesToString, prepareEnvVars, workloadTypes } from "../util"
import { gardenAnnotationKey } from "../../../util/string"
import { RuntimeContext } from "../../../runtime-context"

export const DEFAULT_CPU_REQUEST = "10m"
export const DEFAULT_MEMORY_REQUEST = "64Mi"

export async function deployContainerService(params: DeployServiceParams<ContainerModule>): Promise<ServiceStatus> {
export async function deployContainerService(
params: DeployServiceParams<ContainerModule>,
): Promise<ContainerServiceStatus> {
const { deploymentStrategy } = params.ctx.provider.config

if (deploymentStrategy === "blue-green") {
Expand All @@ -46,17 +47,16 @@ export async function deployContainerService(params: DeployServiceParams<Contain
}

export async function deployContainerServiceRolling(
params: DeployServiceParams<ContainerModule>): Promise<ServiceStatus> {
params: DeployServiceParams<ContainerModule>): Promise<ContainerServiceStatus> {
const { ctx, service, runtimeContext, force, log, hotReload } = params
const k8sCtx = <KubernetesPluginContext>ctx

const namespace = await getAppNamespace(k8sCtx, log, k8sCtx.provider)

const manifests = await createContainerObjects(k8sCtx, log, service, runtimeContext, hotReload)
const { manifests } = await createContainerManifests(k8sCtx, log, service, runtimeContext, hotReload)

// TODO: use Helm instead of kubectl apply
const provider = k8sCtx.provider
const pruneSelector = "service=" + service.name
const pruneSelector = gardenAnnotationKey("service") + "=" + service.name

await apply({ log, provider, manifests, force, namespace, pruneSelector })

Expand All @@ -71,25 +71,15 @@ export async function deployContainerServiceRolling(
return getContainerServiceStatus(params)
}

// Given an array of k8s resources and a Garden service returns matching k8s resource
function getResourcesForService(items: KubernetesServerResource[], service): KubernetesServerResource[] {
return items.filter((resource) => {
return resource.metadata
&& resource.metadata.labels
&& resource.metadata.labels["module"] === service.module.name
&& resource.metadata.labels["service"] === service.name
})
}

export async function deployContainerServiceBlueGreen(
params: DeployServiceParams<ContainerModule>): Promise<ServiceStatus> {
params: DeployServiceParams<ContainerModule>): Promise<ContainerServiceStatus> {

const { ctx, service, runtimeContext, force, log, hotReload } = params
const k8sCtx = <KubernetesPluginContext>ctx
const namespace = await getAppNamespace(k8sCtx, log, k8sCtx.provider)

// Create all the resource manifests for the Garden service which will be deployed
const manifests = await createContainerObjects(k8sCtx, log, service, runtimeContext, hotReload)
const { manifests } = await createContainerManifests(k8sCtx, log, service, runtimeContext, hotReload)

const provider = k8sCtx.provider
const api = await KubeApi.factory(log, provider)
Expand All @@ -116,14 +106,11 @@ export async function deployContainerServiceBlueGreen(
} else {
// A k8s service matching the current Garden service exist in the cluster.
// Proceeding with blue-green deployment
const newVersion = service.module.version.versionString
const versionKey = gardenAnnotationKey("version")

// Remove Service manifest from generated resources
const filteredManifests = manifests.filter(manifest => manifest.kind !== "Service")
// Retrieve new (yet-to-be-deployed) Deployment manifest
const deploymentManifest = find(manifests, (manifest) => {
return manifest.kind === "Deployment"
&& manifest.metadata.labels[gardenAnnotationKey("version")] === service.module.version.versionString
})

// Apply new Deployment manifest (deploy the Green version)
await apply({ log, provider, manifests: filteredManifests, force, namespace })
Expand All @@ -139,12 +126,12 @@ export async function deployContainerServiceBlueGreen(
const servicePatchBody = {
metadata: {
annotations: {
[gardenAnnotationKey("version")]: deploymentManifest.metadata.labels.version,
[versionKey]: newVersion,
},
},
spec: {
selector: {
[gardenAnnotationKey("version")]: deploymentManifest.metadata.labels.version,
[versionKey]: newVersion,
},
},
}
Expand Down Expand Up @@ -175,32 +162,22 @@ export async function deployContainerServiceBlueGreen(

// Clenup unused deployments:
// as a feature we delete all the deployments which don't match any deployed Service.

const deployments = await api.apps.listNamespacedDeployment(namespace)
// Retrieve all unused deployments for current service
const unusedDeployments = getResourcesForService(deployments.items, service)
.filter(deployment => deployment.metadata.labels
&& deployment.metadata.labels[gardenAnnotationKey("version")]
!== deploymentManifest.metadata.labels[gardenAnnotationKey("version")])

if (unusedDeployments) {
// Delete old Deployments (Blue)
await Bluebird.map(
unusedDeployments, oldDeployment => api.apps.deleteNamespacedDeployment(oldDeployment.metadata.name, namespace),
)
await waitForResources({
ctx: k8sCtx,
provider: k8sCtx.provider,
serviceName: `Cleanup deployments`,
resources: manifests,
log,
})
}
log.verbose(`Cleaning up old workloads`)
await deleteObjectsBySelector({
log,
provider,
namespace,
objectTypes: workloadTypes,
// Find workloads that match this service, but have a different version
selector:
`${gardenAnnotationKey("service")}=${service.name},` +
`${versionKey}!=${newVersion}`,
})
}
return getContainerServiceStatus(params)
}

export async function createContainerObjects(
export async function createContainerManifests(
ctx: PluginContext,
log: LogEntry,
service: ContainerService,
Expand All @@ -213,18 +190,19 @@ export async function createContainerObjects(
const namespace = await getAppNamespace(k8sCtx, log, provider)
const api = await KubeApi.factory(log, provider)
const ingresses = await createIngressResources(api, provider, namespace, service)
const deployment = await createDeployment({ provider, service, runtimeContext, namespace, enableHotReload, log })
const workload = await createWorkloadResource({ provider, service, runtimeContext, namespace, enableHotReload, log })
const kubeservices = await createServiceResources(service, namespace)

const objects = [deployment, ...kubeservices, ...ingresses]
const manifests = [workload, ...kubeservices, ...ingresses]

return objects.map(obj => {
for (const obj of manifests) {
set(obj, ["metadata", "labels", gardenAnnotationKey("module")], service.module.name)
set(obj, ["metadata", "labels", gardenAnnotationKey("service")], service.name)
set(obj, ["metadata", "annotations", gardenAnnotationKey("generated")], "true")
set(obj, ["metadata", "annotations", gardenAnnotationKey("version")], version.versionString)
set(obj, ["metadata", "labels", "module"], service.module.name)
set(obj, ["metadata", "labels", "service"], service.name)
return obj
})
}

return { workload, manifests }
}

interface CreateDeploymentParams {
Expand All @@ -236,9 +214,9 @@ interface CreateDeploymentParams {
log: LogEntry,
}

export async function createDeployment(
export async function createWorkloadResource(
{ provider, service, runtimeContext, namespace, enableHotReload, log }: CreateDeploymentParams,
): Promise<KubernetesResource> {
): Promise<KubernetesWorkload> {

const spec = service.spec
let configuredReplicas = service.spec.replicas
Expand Down Expand Up @@ -305,20 +283,10 @@ export async function createDeployment(
container.args = service.spec.args
}

// if (config.entrypoint) {
// container.command = [config.entrypoint]
// }

if (spec.healthCheck) {
configureHealthCheck(container, spec)
}

// if (service.privileged) {
// container.securityContext = {
// privileged: true,
// }
// }

if (spec.volumes && spec.volumes.length) {
configureVolumes(deployment, container, spec)
}
Expand Down Expand Up @@ -400,28 +368,30 @@ export async function createDeployment(
return deployment
}

function deploymentConfig(service: Service, configuredReplicas: number, namespace: string): object {
function getDeploymentName(service: Service) {
return `${service.name}-${service.module.version.versionString}`
}

function deploymentConfig(service: Service, configuredReplicas: number, namespace: string): object {
const labels = {
module: service.module.name,
service: service.name,
[gardenAnnotationKey("module")]: service.module.name,
[gardenAnnotationKey("service")]: service.name,
[gardenAnnotationKey("version")]: service.module.version.versionString,
}

let selector: any = {
let selector = {
matchLabels: {
service: service.name,
[gardenAnnotationKey("service")]: service.name,
[gardenAnnotationKey("version")]: service.module.version.versionString,
},
}

selector.matchLabels[gardenAnnotationKey("version")] = service.module.version.versionString

// TODO: moar type-safety
return {
kind: "Deployment",
apiVersion: "apps/v1",
metadata: {
name: `${service.name}-${service.module.version.versionString}`,
name: getDeploymentName(service),
annotations: {
// we can use this to avoid overriding the replica count if it has been manually scaled
"garden.io/configured.replicas": configuredReplicas.toString(),
Expand All @@ -443,10 +413,6 @@ function deploymentConfig(service: Service, configuredReplicas: number, namespac
restartPolicy: "Always",
terminationGracePeriodSeconds: 5,
dnsPolicy: "ClusterFirst",
// TODO: support private registries
// imagePullSecrets: [
// { name: DOCKER_AUTH_SECRET_NAME },
// ],
volumes: [],
},
},
Expand Down Expand Up @@ -552,45 +518,20 @@ export function rsyncTargetPath(path: string) {
.replace(/\/*$/, "/")
}

export async function deleteService(params: DeleteServiceParams): Promise<ServiceStatus> {
export async function deleteService(params: DeleteServiceParams): Promise<ContainerServiceStatus> {
const { ctx, log, service } = params
const k8sCtx = <KubernetesPluginContext>ctx
const namespace = await getAppNamespace(k8sCtx, log, k8sCtx.provider)
const provider = k8sCtx.provider

await deleteContainerDeployment({ namespace, provider, serviceName: service.name, log })
await deleteObjectsByLabel({
await deleteObjectsBySelector({
log,
provider,
namespace,
labelKey: "service",
labelValue: service.name,
selector: `${gardenAnnotationKey("service")}=${service.name}`,
objectTypes: ["deployment", "replicaset", "pod", "service", "ingress", "daemonset"],
includeUninitialized: false,
})

return { state: "missing" }
}

export async function deleteContainerDeployment(
{ namespace, provider, serviceName, log }:
{ namespace: string, provider: KubernetesProvider, serviceName: string, log: LogEntry },
) {

let found = true
const api = await KubeApi.factory(log, provider)

try {
await api.extensions.deleteNamespacedDeployment(serviceName, namespace, <any>{})
} catch (err) {
if (err.code === 404) {
found = false
} else {
throw err
}
}

if (log) {
found ? log.setSuccess("Service deleted") : log.setWarn("Service not deployed")
}
return { state: "missing", detail: { remoteResources: [], workload: null } }
}
Loading

0 comments on commit 06ed6fd

Please sign in to comment.