diff --git a/README.md b/README.md index 8a71704cd..cc6b8cbf1 100644 --- a/README.md +++ b/README.md @@ -9,32 +9,81 @@ Nowadays, the scheduler is configurable/extendable in the multiple ways: - etc... But, unfortunately, not all configurations/expansions yield good results. -Those who customize the scheduler need to make sure their scheduler is working as expected, and doesn't have an unacceptably negative impact on the scheduling. +Those who customize the scheduler need to make sure their scheduler is working as expected, +and doesn't have an unacceptably negative impact on the scheduling. -In real Kubernetes, we cannot know the results of scheduling in detail without reading the logs, which usually requires privileged access to the control plane. -That's why we are developing a simulator for kube-scheduler -- you can try out the behavior of the scheduler with web UI while checking which plugin made what decision for which Node. +In real Kubernetes, we cannot know the results of scheduling in detail without reading the logs, +which usually requires privileged access to the control plane. + +That's why we are developing a simulator for kube-scheduler +-- you can try out the behavior of the scheduler while checking which plugin made what decision for which Node. ## Simulator's architecture We have several components: -- Simulator (in `/simulator`) -- Web UI (in `/web`) +- Simulator (in `/simulator`): the core implementation of the simulator +- Web UI (in `/web`): the Web client of the simulator - Coming soon... :) (see [./keps](./keps) to see some nice ideas we're working on) ### Simulator -Simulator internally has kube-apiserver, scheduler, and HTTP server. +Simulator is kube-apiserver + scheduler + the HTTP server which mainly for the web UI. + +There are several ways to integrate your scheduler into the simulator. +See [integrate-your-scheduler.md](simulator/docs/integrate-your-scheduler.md). + +You can create any resources by communicating with kube-apiserver in any ways (kubectl, k8s client library, or web UI described next) +and see how scheduling is done. + +When you create Pods, Pods will get annotations from the simulator which contains the scheduling results per plugins or extenders. + +```yaml +kind: Pod +apiVersion: v1 +metadata: + name: hoge-pod + annotations: + scheduler-simulator/bind-result: '{"DefaultBinder":"success"}' + scheduler-simulator/filter-result: >- + {"node-282x7":{"AzureDiskLimits":"passed","EBSLimits":"passed","GCEPDLimits":"passed","InterPodAffinity":"passed","NodeAffinity":"passed","NodeName":"passed","NodePorts":"passed","NodeResourcesFit":"passed","NodeUnschedulable":"passed","NodeVolumeLimits":"passed","PodTopologySpread":"passed","TaintToleration":"passed","VolumeBinding":"passed","VolumeRestrictions":"passed","VolumeZone":"passed"},"node-gp9t4":{"AzureDiskLimits":"passed","EBSLimits":"passed","GCEPDLimits":"passed","InterPodAffinity":"passed","NodeAffinity":"passed","NodeName":"passed","NodePorts":"passed","NodeResourcesFit":"passed","NodeUnschedulable":"passed","NodeVolumeLimits":"passed","PodTopologySpread":"passed","TaintToleration":"passed","VolumeBinding":"passed","VolumeRestrictions":"passed","VolumeZone":"passed"}} + scheduler-simulator/finalscore-result: >- + {"node-282x7":{"ImageLocality":"0","InterPodAffinity":"0","NodeAffinity":"0","NodeNumber":"0","NodeResourcesBalancedAllocation":"76","NodeResourcesFit":"73","PodTopologySpread":"200","TaintToleration":"300","VolumeBinding":"0"},"node-gp9t4":{"ImageLocality":"0","InterPodAffinity":"0","NodeAffinity":"0","NodeNumber":"0","NodeResourcesBalancedAllocation":"76","NodeResourcesFit":"73","PodTopologySpread":"200","TaintToleration":"300","VolumeBinding":"0"}} + scheduler-simulator/permit-result: '{}' + scheduler-simulator/permit-result-timeout: '{}' + scheduler-simulator/postfilter-result: '{}' + scheduler-simulator/prebind-result: '{"VolumeBinding":"success"}' + scheduler-simulator/prefilter-result: '{}' + scheduler-simulator/prefilter-result-status: >- + {"InterPodAffinity":"success","NodeAffinity":"success","NodePorts":"success","NodeResourcesFit":"success","PodTopologySpread":"success","VolumeBinding":"success","VolumeRestrictions":"success"} + scheduler-simulator/prescore-result: >- + {"InterPodAffinity":"success","NodeAffinity":"success","NodeNumber":"success","PodTopologySpread":"success","TaintToleration":"success"} + scheduler-simulator/reserve-result: '{"VolumeBinding":"success"}' + scheduler-simulator/result-history: >- + [{"noderesourcefit-prefilter-data":"{\"MilliCPU\":100,\"Memory\":17179869184,\"EphemeralStorage\":0,\"AllowedPodNumber\":0,\"ScalarResources\":null}","scheduler-simulator/bind-result":"{\"DefaultBinder\":\"success\"}","scheduler-simulator/filter-result":"{\"node-282x7\":{\"AzureDiskLimits\":\"passed\",\"EBSLimits\":\"passed\",\"GCEPDLimits\":\"passed\",\"InterPodAffinity\":\"passed\",\"NodeAffinity\":\"passed\",\"NodeName\":\"passed\",\"NodePorts\":\"passed\",\"NodeResourcesFit\":\"passed\",\"NodeUnschedulable\":\"passed\",\"NodeVolumeLimits\":\"passed\",\"PodTopologySpread\":\"passed\",\"TaintToleration\":\"passed\",\"VolumeBinding\":\"passed\",\"VolumeRestrictions\":\"passed\",\"VolumeZone\":\"passed\"},\"node-gp9t4\":{\"AzureDiskLimits\":\"passed\",\"EBSLimits\":\"passed\",\"GCEPDLimits\":\"passed\",\"InterPodAffinity\":\"passed\",\"NodeAffinity\":\"passed\",\"NodeName\":\"passed\",\"NodePorts\":\"passed\",\"NodeResourcesFit\":\"passed\",\"NodeUnschedulable\":\"passed\",\"NodeVolumeLimits\":\"passed\",\"PodTopologySpread\":\"passed\",\"TaintToleration\":\"passed\",\"VolumeBinding\":\"passed\",\"VolumeRestrictions\":\"passed\",\"VolumeZone\":\"passed\"}}","scheduler-simulator/finalscore-result":"{\"node-282x7\":{\"ImageLocality\":\"0\",\"InterPodAffinity\":\"0\",\"NodeAffinity\":\"0\",\"NodeNumber\":\"0\",\"NodeResourcesBalancedAllocation\":\"76\",\"NodeResourcesFit\":\"73\",\"PodTopologySpread\":\"200\",\"TaintToleration\":\"300\",\"VolumeBinding\":\"0\"},\"node-gp9t4\":{\"ImageLocality\":\"0\",\"InterPodAffinity\":\"0\",\"NodeAffinity\":\"0\",\"NodeNumber\":\"0\",\"NodeResourcesBalancedAllocation\":\"76\",\"NodeResourcesFit\":\"73\",\"PodTopologySpread\":\"200\",\"TaintToleration\":\"300\",\"VolumeBinding\":\"0\"}}","scheduler-simulator/permit-result":"{}","scheduler-simulator/permit-result-timeout":"{}","scheduler-simulator/postfilter-result":"{}","scheduler-simulator/prebind-result":"{\"VolumeBinding\":\"success\"}","scheduler-simulator/prefilter-result":"{}","scheduler-simulator/prefilter-result-status":"{\"InterPodAffinity\":\"success\",\"NodeAffinity\":\"success\",\"NodePorts\":\"success\",\"NodeResourcesFit\":\"success\",\"PodTopologySpread\":\"success\",\"VolumeBinding\":\"success\",\"VolumeRestrictions\":\"success\"}","scheduler-simulator/prescore-result":"{\"InterPodAffinity\":\"success\",\"NodeAffinity\":\"success\",\"NodeNumber\":\"success\",\"PodTopologySpread\":\"success\",\"TaintToleration\":\"success\"}","scheduler-simulator/reserve-result":"{\"VolumeBinding\":\"success\"}","scheduler-simulator/score-result":"{\"node-282x7\":{\"ImageLocality\":\"0\",\"InterPodAffinity\":\"0\",\"NodeAffinity\":\"0\",\"NodeNumber\":\"0\",\"NodeResourcesBalancedAllocation\":\"76\",\"NodeResourcesFit\":\"73\",\"PodTopologySpread\":\"0\",\"TaintToleration\":\"0\",\"VolumeBinding\":\"0\"},\"node-gp9t4\":{\"ImageLocality\":\"0\",\"InterPodAffinity\":\"0\",\"NodeAffinity\":\"0\",\"NodeNumber\":\"0\",\"NodeResourcesBalancedAllocation\":\"76\",\"NodeResourcesFit\":\"73\",\"PodTopologySpread\":\"0\",\"TaintToleration\":\"0\",\"VolumeBinding\":\"0\"}}","scheduler-simulator/selected-node":"node-282x7"}] + scheduler-simulator/score-result: >- + {"node-282x7":{"ImageLocality":"0","InterPodAffinity":"0","NodeAffinity":"0","NodeNumber":"0","NodeResourcesBalancedAllocation":"76","NodeResourcesFit":"73","PodTopologySpread":"0","TaintToleration":"0","VolumeBinding":"0"},"node-gp9t4":{"ImageLocality":"0","InterPodAffinity":"0","NodeAffinity":"0","NodeNumber":"0","NodeResourcesBalancedAllocation":"76","NodeResourcesFit":"73","PodTopologySpread":"0","TaintToleration":"0","VolumeBinding":"0"}} + scheduler-simulator/selected-node: node-282x7 +``` + +You can utilize these results to understand your scheduler, check/test your configurations or customized scheduler's behavior. -You can create any resources by communicating with kube-apiserver via kubectl, k8s client library, or web UI. +Further expansion, you can export internal state more, change specific behaviours on plugins etc by implementing [PluginExtender](./simulator/docs/plugin-extender.md). + +The simulator has its own configuration, +you can refer to the [documentation](./simulator/docs/simulator-server-config.md). See the following docs to know more about simulator: - [how-it-works.md](simulator/docs/how-it-works.md): describes about how the simulator works. - [kube-apiserver.md](simulator/docs/kube-apiserver.md): describe about kube-apiserver in simulator. (how you can configure and access) -- [api.md](simulator/docs/api.md): describes about HTTP server the simulator has. +- [api.md](simulator/docs/api.md): describes about HTTP server the simulator has. (mainly for the webUI) ### Web UI -Web UI is one of the clients of simulator, but it's optimized for simulator. +It's well-optimized Web UI for the simulator. + +Web UI is the best way to check the scheduler's behavior very easily. +Nice table view for the scheduling result, the scheduler configuration reload feature, etc... +every core features of the simulator can be access with human-friendly ways! From the web, you can create/edit/delete these resources to simulate a cluster. @@ -48,7 +97,7 @@ From the web, you can create/edit/delete these resources to simulate a cluster. ![list resources](simulator/docs/images/resources.png) -You can create resources with yaml file as usual. +It has a cool yaml editor to create/edit resources. ![create node](simulator/docs/images/create-node.png) @@ -60,27 +109,14 @@ And, after pods are scheduled, you can see the results of ![result](simulator/docs/images/result.jpg) -You can configure the scheduler on the simulator through KubeSchedulerConfiguration. - -[Scheduler Configuration | Kubernetes](https://kubernetes.io/docs/reference/scheduling/config/) +Also, You can change the configuration of the scheduler through [KubeSchedulerConfiguration](https://kubernetes.io/docs/reference/scheduling/config/). -You can pass a path to KubeSchedulerConfiguration via the environment variable `KUBE_SCHEDULER_CONFIG_PATH` and the simulator will start kube-scheduler with that configuration. - -Note: changes to any fields other than `.profiles` are disabled on simulator, since they do not affect the results of the scheduling. +(Note: changes to any fields other than `.profiles` are disabled on simulator, since they do not affect the results of the scheduling.) ![configure scheduler](simulator/docs/images/schedulerconfiguration.png) -If you want to use your custom plugins as out-of-tree plugins in the simulator, please follow [this doc](simulator/docs/how-to-use-custom-plugins/README.md). - ## Getting started -You can find more information about environment variables available in the simulator server -[here.](./simulator/docs/environment-variables.md) - - -The simulator server configuration can also be set through the configuration file, -you can refer to the [documentation](./simulator/docs/simulator-server-config.md). - ### Run simulator with Docker We have [docker-compose.yml](docker-compose.yml) to run the simulator easily. You should install [docker](https://docs.docker.com/engine/install/) and [docker-compose](https://docs.docker.com/compose/install/) firstly. @@ -101,7 +137,7 @@ Please allocate enough memory in that case. You have to run frontend, server and etcd. -#### Run simulator server and etcd +#### 1. Run simulator server and etcd To run this simulator's server, you have to install Go and etcd. @@ -114,12 +150,13 @@ make start It starts etcd and simulator-server locally. -#### Run simulator frontend +#### 2. Run simulator frontend To run the frontend, please see [README.md](web/README.md) on ./web dir. -## [Beta] Existing cluster importing +## Beta features +### [Beta] Existing cluster importing The simulator can import resources from your cluster. You can use it by setting an `EXTERNAL_IMPORT_ENABLED` environment variable to `1`. diff --git a/simulator/docs/custom-plugin.md b/simulator/docs/custom-plugin.md new file mode 100644 index 000000000..46770a6d5 --- /dev/null +++ b/simulator/docs/custom-plugin.md @@ -0,0 +1,56 @@ +## How to use your custom plugins in the simulator + +This doc describes how to use your custom plugins in the scheduler running in the simulator. + +### 1. Add your custom plugin's registry in OutOfTreeRegistries function. + +Please add your custom plugin's registry in `outOfTreeRegistries` in config package here: + +[kube-scheduler-simulator/simulator/scheduler/config/plugin.go](/simulator/scheduler/config/plugin.go) + +### 2. Configure the scheduler to enable your custom plugin + +You can configure the scheduler to use your custom plugins through KubeSchedulerConfiguration. + +[Scheduler Configuration | Kubernetes](https://kubernetes.io/docs/reference/scheduling/config/) + +You can change the scheduler configuration in Web UI or +by passing a default KubeSchedulerConfiguration file via the environment variable `KUBE_SCHEDULER_CONFIG_PATH`. + +### Example + +We will explain the case where you want to add [nodenumber](../sample/nodenumber/plugin.go) plugin as example. + +The nodenumber plugin is an example plugin that favors nodes that have the number suffix which is the same as the number suffix of the pod name. +And we can configure it via `NodeNumberArgs`. + +First, you need to add registry for the nodenumber plugin to `outOfTreeRegistries`. + +```go +outOfTreeRegistries = runtime.Registry{ + // TODO(user): add your plugins registries here. + nodenumber.Name: nodenumber.New, +} +``` + +Now you can use the nodenumber plugin in the simulator. + +If you apply this configuration to the scheduler, you can see the nodenumber plugin is working (and NodeNumberArgs is applied to the nodenumber plugin) in the simulator, +and see the nodenumber plugin's result like other in-tree plugins. + +```yaml +kind: KubeSchedulerConfiguration +apiVersion: kubescheduler.config.k8s.io/v1 +# .... +profiles: + - schedulerName: default-scheduler + plugins: + # .... + multiPoint: + enabled: + # .... + - name: NodeNumber # added + weight: 10 +``` + +![result](images/result-nodenumber.jpg) \ No newline at end of file diff --git a/simulator/docs/environment-variables.md b/simulator/docs/environment-variables.md index 04441a68c..3278e5e0b 100644 --- a/simulator/docs/environment-variables.md +++ b/simulator/docs/environment-variables.md @@ -1,17 +1,17 @@ -# Environment Variables +## [deprecated] Environment Variables -This page describes the environment variables that are used to configure the simulator. - -Please refer to [docker-compose.yml](./../../docker-compose.yml) as an example use. +**Deprecation notice**: We're planning to remove the configuration via environment variables. +Until deprecation, the simulator will read the configuration in the environment variable first, +if the environment variable is not set, it will read the configuration in the configuration file. +For config file, please refer to the simulator [config.yaml](./../config.yaml). -## For Simulator +--- -**Deprecation notice**: We're planning to remove the configuration via environment variables. +This page describes the environment variables that are used to configure the simulator. -Until deprecation, the simulator will read the configuration in the environment variable first, -if the environment variable is not set, it will read the configuration in the configuration file. +Please refer to [docker-compose.yml](./../../docker-compose.yml) as an example use. -For config file, please refer to the simulator [config.yaml](./../config.yaml). +### For Simulator `PORT`: (required) This is the port number on which kube-scheduler-simulator server is started. diff --git a/simulator/docs/external-scheduler.md b/simulator/docs/external-scheduler.md new file mode 100644 index 000000000..d30e5c9f0 --- /dev/null +++ b/simulator/docs/external-scheduler.md @@ -0,0 +1,90 @@ +## External scheduler + +This document describes how to use the external scheduler instead of the scheduler running in the simulator. + +We use the [`externalscheduler` package](../pkg/externalscheduler); +the scheduler built with the [`externalscheduler` package](../pkg/externalscheduler) will export the scheduling results on each Pod annotation. + +### Use cases + +- Running your scheduler instead of the default one in the simulator + - You can still see the scheduling results in web UI as well! +- Running your scheduler with the simulator feature in your cluster + - All Pods, scheduled by this scheduler, will get the scheduler results on its annotation while each scheduling is done as usual. + - Note that it has performance overhead in each scheduling cycle +since the scheduler needs to make additional effort to export the scheduling results. + +### Change your scheduler + +Here, we assume you're registering your custom plugins in your scheduler like this: + +```go +// your scheduler's main package +func main() { + command := app.NewSchedulerCommand( + app.WithPlugin(yourcustomplugin.Name, yourcustomplugin.New), + ) + + code := cli.Run(command) + os.Exit(code) +} +``` + +Then, you need to replace few lines to use the [`externalscheduler` package](../pkg/externalscheduler). + +```go +func main() { + command, cancelFn, err := externalscheduler.NewSchedulerCommand( + externalscheduler.WithPlugin(yourcustomplugin.Name, yourcustomplugin.New), + externalscheduler.WithPluginExtenders(noderesources.Name, extender.New), // [optional] see plugin-extender.md about PluginExtender. + ) + if err != nil { + klog.Info(fmt.Sprintf("failed to build the scheduler command: %+v", err)) + os.Exit(1) + } + code := cli.Run(command) + cancelFn() + os.Exit(code) +} +``` + +As you see, `externalscheduler.NewSchedulerCommand` has much similar interface as the `app.NewSchedulerCommand`. +You can register your plugins by `externalscheduler.WithPlugin` option. + +Via this step, all Pods scheduled by this scheduler will get the scheduling results in the annotation like in the simulator! + +### Connect your scheduler to the kube-apiserver in the simulator + +If you are here to run the scheduler built with [`externalscheduler` package](../pkg/externalscheduler) in your cluster, +you don't need to follow this step. + +Let's connect your scheduler into the simulator. + +First, you need to set `externalSchedulerEnabled: true` on [the simulator config](../config.yaml) +so that the scheduler in the simulator won't get started. + +Next, you need to connect your scheduler into the simulator's kube-apiserver via KubeSchedulerConfig: + +```yaml +kind: KubeSchedulerConfiguration +apiVersion: kubescheduler.config.k8s.io/v1 +clientConnection: + kubeconfig: ./path/to/kubeconfig.yaml +``` + +You can use this [kubeconfig.yaml](./kubeconfig.yaml) to communicate with the simulator's kube-apiserver. + +### The example external scheduler + +We have the sample external scheduler implementation in [./sample/external-scheduler](./sample/external-scheduler). + +prerequisite: +1. set `externalSchedulerEnabled: true` on [the simulator config](../config.yaml) +2. run the simulator + +```shell +cd sample/external-scheduler +go run main.go --config scheduler.yaml +``` + +You'll see the simulator is working with the external scheduler. \ No newline at end of file diff --git a/simulator/docs/how-it-works.md b/simulator/docs/how-it-works.md index 6f6c562c6..f2eb1246e 100644 --- a/simulator/docs/how-it-works.md +++ b/simulator/docs/how-it-works.md @@ -1,18 +1,18 @@ -# How the simulator works +## How the simulator works -This page describes how this kube-scheduler-simulator works. +This page describes how the simulator works. -## 0. starts the simulator. +### 0. starts the simulator. The simulator server works with the following: - [kube-apiserver (+ etcd)](kube-apiserver.md) - scheduler -- pv controller +- controllers for core resources - [HTTP server](api.md) When the simulator server starts, it will start these components with server. -## 1. users request creating resource. +### 1. users request creating resource. Users can create resources below by communicating with kube-apiserver in simulator via any clients (e.g. kubectl, k8s client library or Web UI) @@ -22,17 +22,18 @@ Users can create resources below by communicating with kube-apiserver in simulat - Persistent Volume Claims - Storage Classes - Priority Classes +- Namespaces -## 2. the scheduler schedules a new pod. +### 2. the scheduler schedules a new pod. When a new pod is created through kube-apiserver, the scheduler starts scheduling. -## 3. the results of score/filter plugins are recorded. +### 3. the results of score/filter plugins are recorded. Normally, when score/filter plugins are called from scheduler, they will calculate the results and return results to the scheduler. But, in the simulator, custom plugins, that behave as score/filter plugin but records result after calculation, are used in scheduler. -## 4. the scheduler binds the pod to a node. +### 4. the scheduler binds the pod to a node. The scheduler finally binds the pod to a node if succeeded, or move the pod back to queue if failed. diff --git a/simulator/docs/how-to-use-custom-plugins/README.md b/simulator/docs/how-to-use-custom-plugins/README.md deleted file mode 100644 index 727f77ba6..000000000 --- a/simulator/docs/how-to-use-custom-plugins/README.md +++ /dev/null @@ -1,191 +0,0 @@ -# How to use your custom plugins in the simulator - -This doc describes how to use your custom plugins in the simulator. - -## 1. Add your custom plugin's registry in OutOfTreeRegistries function. - -Please add your custom plugin's registry in `OutOfTreeRegistries` function in config package here: - -[kube-scheduler-simulator/simulator/scheduler/config/plugin.go](/simulator/scheduler/config/plugin.go) - - -## 2. Add your custom plugin in OutOfTreeFilterPlugins or OutOfTreeScorePlugins function. - -**Only `Filter` and `Score` plugins** - -Please add your custom plugin in `OutOfTreeFilterPlugins` function or `OutOfTreeScorePlugins` function in config package here: - -[kube-scheduler-simulator/simulator/scheduler/config/plugin.go](/simulator/scheduler/config/plugin.go) - -## 3. Configure the scheduler to enable your custom plugin - -You can configure the scheduler to use your custom plugins through KubeSchedulerConfiguration. - -[Scheduler Configuration | Kubernetes](https://kubernetes.io/docs/reference/scheduling/config/) - -You can change the scheduler configuration in Web UI or by passing a KubeSchedulerConfiguration file via the environment variable `KUBE_SCHEDULER_CONFIG_PATH`. - -## Example - -We will explain the case where you want to add [nodenumber](nodenumber/plugin.go) plugin as example. - -The nodenumber plugin is an example plugin that favors nodes that have the number suffix which is the same as the number suffix of the pod name. -And we can configure it via `NodeNumberArgs`. - -First, we need to add registry for the nodenumber plugin to `OutOfTreeRegistries`. - -```go -func OutOfTreeRegistries() runtime.Registry { - return runtime.Registry{ - // Note: add your original score plugins registries here. - nodenumber.Name: nodenumber.New, - } -} -``` - -Next, add nodenumber to `OutOfTreeScorePlugins` since the nodenumber plugin is a Score plugin. - -```go -func OutOfTreeScorePlugins() []v1.Plugin { - return []v1.Plugin{ - // Note: add your original score plugins here. - { - Name: nodenumber.Name, - }, - } -} -``` - -Yes! Now we can use the nodenumber plugin in the simulator! - -If you apply this configuration to the scheduler, you can see the nodenumber plugin is working (and NodeNumberArgs is applied to the nodenumber plugin) in the simulator, -and see the nodenumber plugin's result like other in-tree plugins. - -```yaml -kind: KubeSchedulerConfiguration -apiVersion: kubescheduler.config.k8s.io/v1 -profiles: - - schedulerName: default-scheduler - plugins: - queueSort: - enabled: - - name: PrioritySort - preFilter: - enabled: - - name: NodeResourcesFit - - name: NodePorts - - name: VolumeRestrictions - - name: PodTopologySpread - - name: InterPodAffinity - - name: VolumeBinding - - name: NodeAffinity - filter: - enabled: - - name: NodeUnschedulable - - name: NodeName - - name: TaintToleration - - name: NodeAffinity - - name: NodePorts - - name: NodeResourcesFit - - name: VolumeRestrictions - - name: EBSLimits - - name: GCEPDLimits - - name: NodeVolumeLimits - - name: AzureDiskLimits - - name: VolumeBinding - - name: VolumeZone - - name: PodTopologySpread - - name: InterPodAffinity - postFilter: - enabled: - - name: DefaultPreemption - preScore: - enabled: - - name: InterPodAffinity - - name: PodTopologySpread - - name: TaintToleration - - name: NodeAffinity - - name: NodeNumber - score: - enabled: - - name: NodeResourcesBalancedAllocation - weight: 1 - - name: ImageLocality - weight: 1 - - name: InterPodAffinity - weight: 1 - - name: NodeResourcesFit - weight: 1 - - name: NodeAffinity - weight: 1 - - name: PodTopologySpread - weight: 2 - - name: NodeNumber - weight: 10 - - name: TaintToleration - weight: 1 - reserve: - enabled: - - name: VolumeBinding - permit: {} - preBind: - enabled: - - name: VolumeBinding - bind: - enabled: - - name: DefaultBinder - postBind: {} - pluginConfig: - - name: DefaultPreemption - args: - kind: DefaultPreemptionArgs - apiVersion: kubescheduler.config.k8s.io/v1 - minCandidateNodesPercentage: 10 - minCandidateNodesAbsolute: 100 - - name: InterPodAffinity - args: - kind: InterPodAffinityArgs - apiVersion: kubescheduler.config.k8s.io/v1 - hardPodAffinityWeight: 1 - - name: NodeAffinity - args: - kind: NodeAffinityArgs - apiVersion: kubescheduler.config.k8s.io/v1 - - name: NodeResourcesBalancedAllocation - args: - kind: NodeResourcesBalancedAllocationArgs - apiVersion: kubescheduler.config.k8s.io/v1 - resources: - - name: cpu - weight: 1 - - name: memory - weight: 1 - - name: NodeResourcesFit - args: - kind: NodeResourcesFitArgs - apiVersion: kubescheduler.config.k8s.io/v1 - scoringStrategy: - type: LeastAllocated - resources: - - name: cpu - weight: 1 - - name: memory - weight: 1 - - name: PodTopologySpread - args: - kind: PodTopologySpreadArgs - apiVersion: kubescheduler.config.k8s.io/v1 - defaultingType: System - - name: VolumeBinding - args: - kind: VolumeBindingArgs - apiVersion: kubescheduler.config.k8s.io/v1 - bindTimeoutSeconds: 600 - - name: NodeNumber - args: - kind: NodeNumberArgs - apiVersion: kubescheduler.config.k8s.io/v1 - reverse: true -``` - -![result](result.jpg) \ No newline at end of file diff --git a/simulator/docs/how-to-use-custom-plugins/result.jpg b/simulator/docs/images/result-nodenumber.jpg similarity index 100% rename from simulator/docs/how-to-use-custom-plugins/result.jpg rename to simulator/docs/images/result-nodenumber.jpg diff --git a/simulator/docs/integrate-your-scheduler.md b/simulator/docs/integrate-your-scheduler.md new file mode 100644 index 000000000..83232f6ed --- /dev/null +++ b/simulator/docs/integrate-your-scheduler.md @@ -0,0 +1,13 @@ +## Integrate your scheduler + +There are several ways to integrate your scheduler into the simulator. + +Basically you have two options on the table: +- Integrate your scheduler plugins into the scheduler running in the simulator + - Check out [custom-plugin.md](./custom-plugin.md) +- Disable the scheduler running in the simulator and use your scheduler instead + - We call this feature "external scheduler" + - Check out [external-scheduler.md](./external-scheduler.md) + +Also, if you just want to use your `KubeSchedulerConfig` while using default plugins, +you don't need to follow this page. Check out [simulator-server-config.md](./simulator-server-config.md) instead. diff --git a/simulator/docs/kube-apiserver.md b/simulator/docs/kube-apiserver.md index 17962efc1..1b53a7547 100644 --- a/simulator/docs/kube-apiserver.md +++ b/simulator/docs/kube-apiserver.md @@ -1,35 +1,20 @@ -# Kube-apiserver +## Kube-apiserver This page describes about kube-apiserver run in simulator. -## How to communicate with this kube-apiserver +### How to communicate with this kube-apiserver You can use any ways like kubectl, k8s client library, or our Web UI. The endpoint is "http://localhost:3131" by default. (can be configured by env described in the below section.) -### kubeconfig +#### kubeconfig -You can use this kubeconfig to communicate. +You can use this kubeconfig to communicate with kube-apiserver in the simulator. -```yaml -apiVersion: v1 -kind: Config +[kubeconfig.yaml](./kubeconfig.yaml) -clusters: -- cluster: - server: http://localhost:3131 - name: simulator - -contexts: -- context: - cluster: simulator - name: simulator - -current-context: simulator -``` - -### kubectl +#### kubectl You can use the `--server` option. @@ -39,14 +24,14 @@ kubectl get pods --server=localhost:3131 Of course, you can also use the above kubeconfig. -## How it is configured +### How it is configured -### Environment Variable +#### Environment Variable The kube-apiserver is configured to expose on the port `KUBE_API_PORT` and on the network interface `KUBE_API_HOST`. If the two variables are not specified, port `3131` will be used with the localhost `127.0.0.1` address. -### Server Creation +#### Server Creation We create a kube-apiserver instance by utilising the code path in `Kubernetes/cmd/kube-apiserver`, meaning we do not have to maintain any apiserver code. diff --git a/simulator/docs/kubeconfig.yaml b/simulator/docs/kubeconfig.yaml new file mode 100644 index 000000000..58689917f --- /dev/null +++ b/simulator/docs/kubeconfig.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: Config + +clusters: + - cluster: + server: http://localhost:3131 + name: simulator + +contexts: + - context: + cluster: simulator + name: simulator + +current-context: simulator diff --git a/simulator/docs/plugin-extender.md b/simulator/docs/plugin-extender.md new file mode 100644 index 000000000..04a831dc0 --- /dev/null +++ b/simulator/docs/plugin-extender.md @@ -0,0 +1,109 @@ +## Plugin extenders + +**The plugin extender can be used only with [the external scheduler](./external-scheduler.md).** + +The simulator has the concept "Plugin Extenders" which allows you to: +- export plugin's internal state more +- change specific behaviours on particular plugin by injecting the result +- etc... + +(Note that it's not related to the scheduler's webhook which is also called "extender". +(Sorry for the confusing name 😅)) + +The Plugin Extenders has `BeforeXXX` and `AfterXXX` for each extension point. (XXX = any extension points. e.g., Filter, Score..etc) + +For example, `BeforeFilter` is literally called before Filter plugin, +and `AfterFilter` func is called after Filter plugin. + +There are multiple interfaces named `XXXXPluginExtender`. + +```go +// FilterPluginExtender is the extender for Filter plugin. +type FilterPluginExtender interface { + // BeforeFilter is a function that runs before the Filter method of the original plugin. + // If BeforeFilter returns non-success status, the simulator plugin doesn't run the Filter method of the original plugin and return that status. + BeforeFilter(ctx context.Context, state *framework.CycleState, pod *v1.Pod, nodeInfo *framework.NodeInfo) *framework.Status + // AfterFilter is a function that is run after the Filter method of the original plugin. + // A Filter of the simulator plugin finally returns the status returned from AfterFilter. + AfterFilter(ctx context.Context, state *framework.CycleState, pod *v1.Pod, nodeInfo *framework.NodeInfo, filterResult *framework.Status) *framework.Status +} +``` + +### export something in each Pod's annotation via `SimulatorHandle` + +Each PluginExtender can have `SimulatorHandle`, and you can export some internal state through `SimulatorHandle`. + +Example: + +```go +func (e *noderesourcefitPreFilterPluginExtender) AfterPreFilter(ctx context.Context, state *framework.CycleState, pod *v1.Pod, preFilterResult *framework.PreFilterResult, preFilterStatus *framework.Status) (*framework.PreFilterResult, *framework.Status) { + // see ./sample/extender/extender.go + //... + e.handle.AddCustomResult(pod.Namespace, pod.Name, "noderesourcefit-prefilter-data", prefilterData) +} +``` + +If you use the above extender, +each Pod will get `"noderesourcefit-prefilter-data": prefilterData` annotation in each scheduling like other scheduling results. + +### use plugin extender + +**Currently, the plugin extender can be used only in [the external scheduler](./external-scheduler.md).** + +You can use `externalscheduler.WithPluginExtenders` option in `externalscheduler.NewSchedulerCommand` +to enable some PluginExtender in particular plugin. + +```go +func main() { + command, cancelFn, err := externalscheduler.NewSchedulerCommand( + externalscheduler.WithPluginExtenders(noderesources.Name, extender.New), + ) + if err != nil { + klog.Info(fmt.Sprintf("failed to build the scheduler command: %+v", err)) + os.Exit(1) + } + code := cli.Run(command) + cancelFn() + os.Exit(code) +} +``` + +### The example plugin extender + +We have the sample plugin extender implementation in [./sample/extender](./sample/plugin-extender). + +Please follow [this](./external-scheduler.md#the-example-external-scheduler) +to see how this sample plugin extender works with the external scheduler. + +You will see each Pod gets `noderesourcefit-prefilter-data` annotation along with other scheduling results like this: + +```yaml +kind: Pod +apiVersion: v1 +metadata: + name: pod-8ldq5 + namespace: default + annotations: + noderesourcefit-prefilter-data: >- + {"MilliCPU":100,"Memory":17179869184,"EphemeralStorage":0,"AllowedPodNumber":0,"ScalarResources":null} + scheduler-simulator/bind-result: '{"DefaultBinder":"success"}' + scheduler-simulator/filter-result: >- + {"node-282x7":{"AzureDiskLimits":"passed","EBSLimits":"passed","GCEPDLimits":"passed","InterPodAffinity":"passed","NodeAffinity":"passed","NodeName":"passed","NodePorts":"passed","NodeResourcesFit":"passed","NodeUnschedulable":"passed","NodeVolumeLimits":"passed","PodTopologySpread":"passed","TaintToleration":"passed","VolumeBinding":"passed","VolumeRestrictions":"passed","VolumeZone":"passed"},"node-gp9t4":{"AzureDiskLimits":"passed","EBSLimits":"passed","GCEPDLimits":"passed","InterPodAffinity":"passed","NodeAffinity":"passed","NodeName":"passed","NodePorts":"passed","NodeResourcesFit":"passed","NodeUnschedulable":"passed","NodeVolumeLimits":"passed","PodTopologySpread":"passed","TaintToleration":"passed","VolumeBinding":"passed","VolumeRestrictions":"passed","VolumeZone":"passed"}} + scheduler-simulator/finalscore-result: >- + {"node-282x7":{"ImageLocality":"0","InterPodAffinity":"0","NodeAffinity":"0","NodeNumber":"0","NodeResourcesBalancedAllocation":"52","NodeResourcesFit":"47","PodTopologySpread":"200","TaintToleration":"300","VolumeBinding":"0"},"node-gp9t4":{"ImageLocality":"0","InterPodAffinity":"0","NodeAffinity":"0","NodeNumber":"0","NodeResourcesBalancedAllocation":"76","NodeResourcesFit":"73","PodTopologySpread":"200","TaintToleration":"300","VolumeBinding":"0"}} + scheduler-simulator/permit-result: '{}' + scheduler-simulator/permit-result-timeout: '{}' + scheduler-simulator/postfilter-result: '{}' + scheduler-simulator/prebind-result: '{"VolumeBinding":"success"}' + scheduler-simulator/prefilter-result: '{}' + scheduler-simulator/prefilter-result-status: >- + {"InterPodAffinity":"success","NodeAffinity":"success","NodePorts":"success","NodeResourcesFit":"success","PodTopologySpread":"success","VolumeBinding":"success","VolumeRestrictions":"success"} + scheduler-simulator/prescore-result: >- + {"InterPodAffinity":"success","NodeAffinity":"success","NodeNumber":"success","PodTopologySpread":"success","TaintToleration":"success"} + scheduler-simulator/reserve-result: '{"VolumeBinding":"success"}' + scheduler-simulator/result-history: >- + [{"noderesourcefit-prefilter-data":"{\"MilliCPU\":100,\"Memory\":17179869184,\"EphemeralStorage\":0,\"AllowedPodNumber\":0,\"ScalarResources\":null}","scheduler-simulator/bind-result":"{\"DefaultBinder\":\"success\"}","scheduler-simulator/filter-result":"{\"node-282x7\":{\"AzureDiskLimits\":\"passed\",\"EBSLimits\":\"passed\",\"GCEPDLimits\":\"passed\",\"InterPodAffinity\":\"passed\",\"NodeAffinity\":\"passed\",\"NodeName\":\"passed\",\"NodePorts\":\"passed\",\"NodeResourcesFit\":\"passed\",\"NodeUnschedulable\":\"passed\",\"NodeVolumeLimits\":\"passed\",\"PodTopologySpread\":\"passed\",\"TaintToleration\":\"passed\",\"VolumeBinding\":\"passed\",\"VolumeRestrictions\":\"passed\",\"VolumeZone\":\"passed\"},\"node-gp9t4\":{\"AzureDiskLimits\":\"passed\",\"EBSLimits\":\"passed\",\"GCEPDLimits\":\"passed\",\"InterPodAffinity\":\"passed\",\"NodeAffinity\":\"passed\",\"NodeName\":\"passed\",\"NodePorts\":\"passed\",\"NodeResourcesFit\":\"passed\",\"NodeUnschedulable\":\"passed\",\"NodeVolumeLimits\":\"passed\",\"PodTopologySpread\":\"passed\",\"TaintToleration\":\"passed\",\"VolumeBinding\":\"passed\",\"VolumeRestrictions\":\"passed\",\"VolumeZone\":\"passed\"}}","scheduler-simulator/finalscore-result":"{\"node-282x7\":{\"ImageLocality\":\"0\",\"InterPodAffinity\":\"0\",\"NodeAffinity\":\"0\",\"NodeNumber\":\"0\",\"NodeResourcesBalancedAllocation\":\"52\",\"NodeResourcesFit\":\"47\",\"PodTopologySpread\":\"200\",\"TaintToleration\":\"300\",\"VolumeBinding\":\"0\"},\"node-gp9t4\":{\"ImageLocality\":\"0\",\"InterPodAffinity\":\"0\",\"NodeAffinity\":\"0\",\"NodeNumber\":\"0\",\"NodeResourcesBalancedAllocation\":\"76\",\"NodeResourcesFit\":\"73\",\"PodTopologySpread\":\"200\",\"TaintToleration\":\"300\",\"VolumeBinding\":\"0\"}}","scheduler-simulator/permit-result":"{}","scheduler-simulator/permit-result-timeout":"{}","scheduler-simulator/postfilter-result":"{}","scheduler-simulator/prebind-result":"{\"VolumeBinding\":\"success\"}","scheduler-simulator/prefilter-result":"{}","scheduler-simulator/prefilter-result-status":"{\"InterPodAffinity\":\"success\",\"NodeAffinity\":\"success\",\"NodePorts\":\"success\",\"NodeResourcesFit\":\"success\",\"PodTopologySpread\":\"success\",\"VolumeBinding\":\"success\",\"VolumeRestrictions\":\"success\"}","scheduler-simulator/prescore-result":"{\"InterPodAffinity\":\"success\",\"NodeAffinity\":\"success\",\"NodeNumber\":\"success\",\"PodTopologySpread\":\"success\",\"TaintToleration\":\"success\"}","scheduler-simulator/reserve-result":"{\"VolumeBinding\":\"success\"}","scheduler-simulator/score-result":"{\"node-282x7\":{\"ImageLocality\":\"0\",\"InterPodAffinity\":\"0\",\"NodeAffinity\":\"0\",\"NodeNumber\":\"0\",\"NodeResourcesBalancedAllocation\":\"52\",\"NodeResourcesFit\":\"47\",\"PodTopologySpread\":\"0\",\"TaintToleration\":\"0\",\"VolumeBinding\":\"0\"},\"node-gp9t4\":{\"ImageLocality\":\"0\",\"InterPodAffinity\":\"0\",\"NodeAffinity\":\"0\",\"NodeNumber\":\"0\",\"NodeResourcesBalancedAllocation\":\"76\",\"NodeResourcesFit\":\"73\",\"PodTopologySpread\":\"0\",\"TaintToleration\":\"0\",\"VolumeBinding\":\"0\"}}","scheduler-simulator/selected-node":"node-gp9t4"}] + scheduler-simulator/score-result: >- + {"node-282x7":{"ImageLocality":"0","InterPodAffinity":"0","NodeAffinity":"0","NodeNumber":"0","NodeResourcesBalancedAllocation":"52","NodeResourcesFit":"47","PodTopologySpread":"0","TaintToleration":"0","VolumeBinding":"0"},"node-gp9t4":{"ImageLocality":"0","InterPodAffinity":"0","NodeAffinity":"0","NodeNumber":"0","NodeResourcesBalancedAllocation":"76","NodeResourcesFit":"73","PodTopologySpread":"0","TaintToleration":"0","VolumeBinding":"0"}} + scheduler-simulator/selected-node: node-gp9t4 +``` diff --git a/simulator/docs/sample/external-scheduler/kubeconfig.yaml b/simulator/docs/sample/external-scheduler/kubeconfig.yaml new file mode 100644 index 000000000..58689917f --- /dev/null +++ b/simulator/docs/sample/external-scheduler/kubeconfig.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: Config + +clusters: + - cluster: + server: http://localhost:3131 + name: simulator + +contexts: + - context: + cluster: simulator + name: simulator + +current-context: simulator diff --git a/simulator/docs/sample/external-scheduler/main.go b/simulator/docs/sample/external-scheduler/main.go new file mode 100644 index 000000000..df8dad541 --- /dev/null +++ b/simulator/docs/sample/external-scheduler/main.go @@ -0,0 +1,33 @@ +package main + +import ( + "fmt" + "os" + + _ "k8s.io/client-go/plugin/pkg/client/auth/gcp" + "k8s.io/component-base/cli" + _ "k8s.io/component-base/logs/json/register" // for JSON log format registration + _ "k8s.io/component-base/metrics/prometheus/clientgo" + _ "k8s.io/component-base/metrics/prometheus/version" // for version metric registration + "k8s.io/klog/v2" + "k8s.io/kubernetes/pkg/scheduler/framework/plugins/noderesources" + + "sigs.k8s.io/kube-scheduler-simulator/simulator/docs/sample/nodenumber" + extender "sigs.k8s.io/kube-scheduler-simulator/simulator/docs/sample/plugin-extender" + "sigs.k8s.io/kube-scheduler-simulator/simulator/pkg/externalscheduler" +) + +func main() { + command, cancelFn, err := externalscheduler.NewSchedulerCommand( + externalscheduler.WithPlugin(nodenumber.Name, nodenumber.New), + externalscheduler.WithPluginExtenders(noderesources.Name, extender.New), + ) + if err != nil { + klog.Info(fmt.Sprintf("failed to build the scheduler command: %+v", err)) + os.Exit(1) + } + code := cli.Run(command) + + cancelFn() + os.Exit(code) +} diff --git a/simulator/docs/sample/external-scheduler/scheduler.yaml b/simulator/docs/sample/external-scheduler/scheduler.yaml new file mode 100644 index 000000000..e4642be45 --- /dev/null +++ b/simulator/docs/sample/external-scheduler/scheduler.yaml @@ -0,0 +1,11 @@ +kind: KubeSchedulerConfiguration +apiVersion: kubescheduler.config.k8s.io/v1 +clientConnection: + kubeconfig: kubeconfig.yaml +profiles: + - schedulerName: default-scheduler + plugins: + multiPoint: + enabled: + - name: NodeNumber + weight: 10 diff --git a/simulator/docs/how-to-use-custom-plugins/nodenumber/plugin.go b/simulator/docs/sample/nodenumber/plugin.go similarity index 96% rename from simulator/docs/how-to-use-custom-plugins/nodenumber/plugin.go rename to simulator/docs/sample/nodenumber/plugin.go index 4fe9b20e4..5cf26b1b8 100644 --- a/simulator/docs/how-to-use-custom-plugins/nodenumber/plugin.go +++ b/simulator/docs/sample/nodenumber/plugin.go @@ -57,6 +57,8 @@ func (s *preScoreState) Clone() framework.StateData { } func (pl *NodeNumber) PreScore(ctx context.Context, state *framework.CycleState, pod *v1.Pod, nodes []*v1.Node) *framework.Status { + klog.Info("execute PreScore on NodeNumber plugin", "pod", klog.KObj(pod)) + podNameLastChar := pod.Name[len(pod.Name)-1:] podnum, err := strconv.Atoi(podNameLastChar) if err != nil { @@ -82,6 +84,7 @@ var ErrNotExpectedPreScoreState = errors.New("unexpected pre score state") // Score invoked at the score extension point. func (pl *NodeNumber) Score(ctx context.Context, state *framework.CycleState, pod *v1.Pod, nodeName string) (int64, *framework.Status) { + klog.Info("execute Score on NodeNumber plugin", "pod", klog.KObj(pod)) data, err := state.Read(preScoreStateKey) if err != nil { // return success even if there is no value in preScoreStateKey, since the diff --git a/simulator/docs/sample/plugin-extender/extender.go b/simulator/docs/sample/plugin-extender/extender.go new file mode 100644 index 000000000..4544db91e --- /dev/null +++ b/simulator/docs/sample/plugin-extender/extender.go @@ -0,0 +1,74 @@ +package extender + +import ( + "context" + "encoding/json" + + v1 "k8s.io/api/core/v1" + "k8s.io/klog/v2" + "k8s.io/kubernetes/pkg/scheduler/framework" + "k8s.io/kubernetes/pkg/scheduler/framework/plugins/noderesources" + + "sigs.k8s.io/kube-scheduler-simulator/simulator/scheduler/plugin" + "sigs.k8s.io/kube-scheduler-simulator/simulator/util" +) + +type noderesourcefitPreFilterPluginExtender struct { + handle plugin.SimulatorHandle +} + +// New initializes noderesourcefitPreFilterPluginExtender in plugin.PluginExtenders. +func New(handle plugin.SimulatorHandle) plugin.PluginExtenders { + e := &noderesourcefitPreFilterPluginExtender{ + handle: handle, + } + + return plugin.PluginExtenders{PreFilterPluginExtender: e} // only PreFilterPluginExtender +} + +func (e *noderesourcefitPreFilterPluginExtender) BeforePreFilter(ctx context.Context, state *framework.CycleState, pod *v1.Pod) (*framework.PreFilterResult, *framework.Status) { + klog.Info("execute BeforePreFilter on noderesourcefitPreFilterPluginExtender", "pod", klog.KObj(pod)) + // do nothing. + return nil, nil +} + +// AfterPreFilter checks what noderesource plugin stores into the cyclestate for this scheduling, +// and store it through the SimulatorHandle. +// +// By this func, each Pod will get noderesourcefit-prefilter-data after scheduling like: +// --- +// kind: Pod +// apiVersion: v1 +// metadata: +// +// name: pod-8ldq5 +// namespace: default +// annotations: +// noderesourcefit-prefilter-data: >- +// {"MilliCPU":100,"Memory":17179869184,"EphemeralStorage":0,"AllowedPodNumber":0,"ScalarResources":null} +func (e *noderesourcefitPreFilterPluginExtender) AfterPreFilter(ctx context.Context, state *framework.CycleState, pod *v1.Pod, preFilterResult *framework.PreFilterResult, preFilterStatus *framework.Status) (*framework.PreFilterResult, *framework.Status) { + klog.Info("execute AfterPreFilter on noderesourcefitPreFilterPluginExtender", "pod", klog.KObj(pod)) + + c, err := state.Read("PreFilter" + noderesources.Name) + if err != nil { + klog.Info("no state data", "pod", klog.KObj(pod)) + return preFilterResult, preFilterStatus + } + + // use util.PrivateFieldsDecoder to access private fields of c. + value := util.PrivateFieldsDecoder(c, "Resource") + data := value.Interface().(framework.Resource) + + j, err := json.Marshal(data) + if err != nil { + klog.Info("json marshal failed in extender", "pod", klog.KObj(pod)) + return preFilterResult, preFilterStatus + } + + prefilterData := string(j) + + // store data via plugin.SimulatorHandle + e.handle.AddCustomResult(pod.Namespace, pod.Name, "noderesourcefit-prefilter-data", prefilterData) + + return preFilterResult, preFilterStatus +}