From 81367a36bb433b6f5beb2dcb65c807372c301f38 Mon Sep 17 00:00:00 2001 From: jaintpharsha Date: Mon, 9 May 2022 08:10:07 +0530 Subject: [PATCH 01/37] jenkins env 09-May-22 --- pipeline/Jenkinsfile | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/pipeline/Jenkinsfile b/pipeline/Jenkinsfile index 22e3b78..d455d19 100644 --- a/pipeline/Jenkinsfile +++ b/pipeline/Jenkinsfile @@ -2,12 +2,19 @@ pipeline { agent { label 'slave2' } + environment { + TEST = "pipeline_level" + } + stages { stage('Build') { + environment { + TEST = "stage_level" + } steps { sh ''' - echo "STAGE 1: This is a build stage" + echo "STAGE 1: This is a build stage $TEST" sleep 5 ''' } @@ -19,7 +26,7 @@ pipeline { stage('Test') { steps { sh ''' - echo "STAGE 2: This is a Test stage" + echo "STAGE 2: This is a Test stage $TEST" sleep 5 ''' } @@ -28,7 +35,7 @@ pipeline { stage('Deploy') { steps { sh ''' - echo "STAGE 3: This is a Deploy stage" + echo "STAGE 3: This is a Deploy stage $TEST" sleep 5 ''' } From 25de063d0f6a0725998afff12569d374f129f2af Mon Sep 17 00:00:00 2001 From: jaintpharsha Date: Mon, 9 May 2022 08:14:47 +0530 Subject: [PATCH 02/37] jenkins env 09-May-22 --- pipeline/Jenkinsfile | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pipeline/Jenkinsfile b/pipeline/Jenkinsfile index d455d19..dacbc6a 100644 --- a/pipeline/Jenkinsfile +++ b/pipeline/Jenkinsfile @@ -21,8 +21,12 @@ pipeline { } stage('Test and Deploy') { + + input { + message 'Press yes to continue' + } + parallel { - stage('Test') { steps { sh ''' From be555e35ae1374df395d4fa0a427d361f6a73d34 Mon Sep 17 00:00:00 2001 From: jaintpharsha Date: Mon, 9 May 2022 08:25:38 +0530 Subject: [PATCH 03/37] jenkins parameters 09-May-22 --- pipeline/Jenkinsfile | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/pipeline/Jenkinsfile b/pipeline/Jenkinsfile index dacbc6a..0bfd176 100644 --- a/pipeline/Jenkinsfile +++ b/pipeline/Jenkinsfile @@ -2,16 +2,16 @@ pipeline { agent { label 'slave2' } - environment { - TEST = "pipeline_level" - } + parameters { + string defaultValue: 'test', description: 'Which environment should the build need to deployed', name: 'TEST', trim: true + } + + stages { stage('Build') { - environment { - TEST = "stage_level" - } + steps { sh ''' echo "STAGE 1: This is a build stage $TEST" From d12eed4205cf1f886dc7c1a435e580d77fbb7ed0 Mon Sep 17 00:00:00 2001 From: jaintpharsha Date: Mon, 9 May 2022 08:34:41 +0530 Subject: [PATCH 04/37] jenkins catchError 09-May-22 --- jenkins.txt | 3 ++- pipeline/Jenkinsfile | 23 +++++++---------------- 2 files changed, 9 insertions(+), 17 deletions(-) diff --git a/jenkins.txt b/jenkins.txt index 2a0e7bb..afba352 100644 --- a/jenkins.txt +++ b/jenkins.txt @@ -67,4 +67,5 @@ Agent on a perticular node with label. docker - \ No newline at end of file + +Build should not fail if a stage fails. \ No newline at end of file diff --git a/pipeline/Jenkinsfile b/pipeline/Jenkinsfile index 0bfd176..71c0b21 100644 --- a/pipeline/Jenkinsfile +++ b/pipeline/Jenkinsfile @@ -1,36 +1,27 @@ pipeline { agent { label 'slave2' } - - - parameters { - string defaultValue: 'test', description: 'Which environment should the build need to deployed', name: 'TEST', trim: true - } - stages { stage('Build') { steps { - sh ''' - echo "STAGE 1: This is a build stage $TEST" - sleep 5 - ''' + catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { + sh ''' + exit 0 + ''' + } } } stage('Test and Deploy') { - input { - message 'Press yes to continue' - } - parallel { stage('Test') { steps { sh ''' - echo "STAGE 2: This is a Test stage $TEST" + echo "STAGE 2: This is a Test stage" sleep 5 ''' } @@ -39,7 +30,7 @@ pipeline { stage('Deploy') { steps { sh ''' - echo "STAGE 3: This is a Deploy stage $TEST" + echo "STAGE 3: This is a Deploy stage" sleep 5 ''' } From e263658a1d5c28d16b726fcb71187142ef033979 Mon Sep 17 00:00:00 2001 From: jaintpharsha Date: Mon, 9 May 2022 08:39:35 +0530 Subject: [PATCH 05/37] jenkins when condition 09-May-22 --- pipeline/Jenkinsfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pipeline/Jenkinsfile b/pipeline/Jenkinsfile index 71c0b21..27890b4 100644 --- a/pipeline/Jenkinsfile +++ b/pipeline/Jenkinsfile @@ -5,7 +5,9 @@ pipeline { stages { stage('Build') { - + when { + branch 'release*' + } steps { catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { sh ''' From 67a7050d7e7d2d79447a9d88ed641588ac4b6717 Mon Sep 17 00:00:00 2001 From: jaintpharsha Date: Mon, 9 May 2022 08:43:18 +0530 Subject: [PATCH 06/37] jenkins if condition 09-May-22 --- pipeline/Jenkinsfile | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/pipeline/Jenkinsfile b/pipeline/Jenkinsfile index 27890b4..71002f5 100644 --- a/pipeline/Jenkinsfile +++ b/pipeline/Jenkinsfile @@ -5,15 +5,14 @@ pipeline { stages { stage('Build') { - when { - branch 'release*' - } - steps { - catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { - sh ''' - exit 0 - ''' - } + if (env.BRANCH_NAME == "release*" ) { + steps { + catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { + sh ''' + exit 0 + ''' + } + } } } @@ -23,7 +22,7 @@ pipeline { stage('Test') { steps { sh ''' - echo "STAGE 2: This is a Test stage" + echo "$NODE_NAME -> $JOB_NAME -> $BUILD_NUMBER " sleep 5 ''' } From 2d1a58a0640a92ac59109b6a285d38fda1d9b4e6 Mon Sep 17 00:00:00 2001 From: jaintpharsha Date: Mon, 9 May 2022 08:44:16 +0530 Subject: [PATCH 07/37] jenkins if condition 09-May-22 --- pipeline/Jenkinsfile | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/pipeline/Jenkinsfile b/pipeline/Jenkinsfile index 71002f5..1b49ae7 100644 --- a/pipeline/Jenkinsfile +++ b/pipeline/Jenkinsfile @@ -5,13 +5,14 @@ pipeline { stages { stage('Build') { - if (env.BRANCH_NAME == "release*" ) { - steps { - catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { - sh ''' - exit 0 - ''' - } + steps { + if (env.BRANCH_NAME == "release*" ) { + + catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { + sh ''' + exit 0 + ''' + } } } } From 6db032bc45731b6cbe6d120e8adb77318615317f Mon Sep 17 00:00:00 2001 From: jaintpharsha Date: Mon, 9 May 2022 08:45:13 +0530 Subject: [PATCH 08/37] jenkins if condition 09-May-22 --- pipeline/Jenkinsfile | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/pipeline/Jenkinsfile b/pipeline/Jenkinsfile index 1b49ae7..acae017 100644 --- a/pipeline/Jenkinsfile +++ b/pipeline/Jenkinsfile @@ -5,16 +5,16 @@ pipeline { stages { stage('Build') { - steps { - if (env.BRANCH_NAME == "release*" ) { - - catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { - sh ''' - exit 0 - ''' - } - } + when { + branch 'main' } + steps { + catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { + sh ''' + exit 0 + ''' + } + } } stage('Test and Deploy') { From ae8944c503ab4939eb1eb968ca5e6c2a40f6d3eb Mon Sep 17 00:00:00 2001 From: jaintpharsha Date: Mon, 9 May 2022 09:13:21 +0530 Subject: [PATCH 09/37] jenkins shared lib 09-May-22 --- jenkins.txt | 23 ++++++++++++++++++++++- pipeline/Jenkinsfile | 6 +++++- 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/jenkins.txt b/jenkins.txt index afba352..46243c2 100644 --- a/jenkins.txt +++ b/jenkins.txt @@ -68,4 +68,25 @@ Agent docker -Build should not fail if a stage fails. \ No newline at end of file +Build should not fail if a stage fails. + use catchError for this requirement + + stage('Build') { + steps { + catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { + sh ''' + exit 0 + ''' + } + } + } + +Jenkins shared library set-up + STEP 1: Create a git repo (To keep our groovy scripts and it is called as shared library) + STEP 2: Create a folder by name "vars" in the root of your repo. + STEP 3: Create a script with .groovy type (getUser.groovy). + STEP 4: configure jenkins to attach the above library. + Manage Jenkins -> Configure system -> Global Pipeline Libraries + Add -> Name - give any name + Default Version - Branch in which vars is there + Retrieval method - Modern SCM \ No newline at end of file diff --git a/pipeline/Jenkinsfile b/pipeline/Jenkinsfile index acae017..a49d87b 100644 --- a/pipeline/Jenkinsfile +++ b/pipeline/Jenkinsfile @@ -1,6 +1,10 @@ pipeline { agent { label 'slave2' } + + libraries { + lib('printuser@main') + } stages { @@ -11,7 +15,7 @@ pipeline { steps { catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { sh ''' - exit 0 + getUser 'Harsha' 'Trainee' ''' } } From 6f583e2d2326068e97cae6a522938ff0875849ff Mon Sep 17 00:00:00 2001 From: jaintpharsha Date: Mon, 9 May 2022 09:14:18 +0530 Subject: [PATCH 10/37] jenkins shared lib 09-May-22 --- pipeline/Jenkinsfile | 3 --- 1 file changed, 3 deletions(-) diff --git a/pipeline/Jenkinsfile b/pipeline/Jenkinsfile index a49d87b..438bee3 100644 --- a/pipeline/Jenkinsfile +++ b/pipeline/Jenkinsfile @@ -9,9 +9,6 @@ pipeline { stages { stage('Build') { - when { - branch 'main' - } steps { catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { sh ''' From ad837bb373d003d5b1047df0e0bb0f970d153fd9 Mon Sep 17 00:00:00 2001 From: jaintpharsha Date: Mon, 9 May 2022 09:14:54 +0530 Subject: [PATCH 11/37] jenkins shared lib 09-May-22 --- pipeline/Jenkinsfile | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/pipeline/Jenkinsfile b/pipeline/Jenkinsfile index 438bee3..c4b0723 100644 --- a/pipeline/Jenkinsfile +++ b/pipeline/Jenkinsfile @@ -11,9 +11,7 @@ pipeline { stage('Build') { steps { catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { - sh ''' - getUser 'Harsha' 'Trainee' - ''' + getUser 'Harsha' 'Trainee' } } } From 16f1a5cb0ad8e2931b70a4d6ec1faf1811d0942b Mon Sep 17 00:00:00 2001 From: jaintpharsha Date: Mon, 9 May 2022 09:16:37 +0530 Subject: [PATCH 12/37] jenkins shared lib 09-May-22 --- pipeline/Jenkinsfile | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/pipeline/Jenkinsfile b/pipeline/Jenkinsfile index c4b0723..02ed6a5 100644 --- a/pipeline/Jenkinsfile +++ b/pipeline/Jenkinsfile @@ -1,17 +1,19 @@ +@Library('printuser')_ + +getUser 'Harsha' 'Trainee' + pipeline { agent { label 'slave2' } - libraries { - lib('printuser@main') - } + stages { stage('Build') { steps { catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { - getUser 'Harsha' 'Trainee' + sh 'echo "STAGE 1 this is build stage"' } } } From 4f73b34d59ee6a1a3ddaa253fe9b2a62a92e00db Mon Sep 17 00:00:00 2001 From: jaintpharsha Date: Mon, 9 May 2022 09:20:26 +0530 Subject: [PATCH 13/37] jenkins shared lib 09-May-22 --- pipeline/Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipeline/Jenkinsfile b/pipeline/Jenkinsfile index 02ed6a5..1ad5725 100644 --- a/pipeline/Jenkinsfile +++ b/pipeline/Jenkinsfile @@ -1,6 +1,6 @@ @Library('printuser')_ -getUser 'Harsha' 'Trainee' +getUser 'Harsha', 'Trainee' pipeline { From 11eaa3d5d16dcfe39862db70cf671678fd70c2d4 Mon Sep 17 00:00:00 2001 From: jaintpharsha Date: Mon, 9 May 2022 09:24:13 +0530 Subject: [PATCH 14/37] jenkins shared lib 09-May-22 --- jenkins.txt | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/jenkins.txt b/jenkins.txt index 46243c2..f5c02d2 100644 --- a/jenkins.txt +++ b/jenkins.txt @@ -87,6 +87,11 @@ Jenkins shared library set-up STEP 3: Create a script with .groovy type (getUser.groovy). STEP 4: configure jenkins to attach the above library. Manage Jenkins -> Configure system -> Global Pipeline Libraries - Add -> Name - give any name + Add -> Name - give any name () Default Version - Branch in which vars is there - Retrieval method - Modern SCM \ No newline at end of file + Retrieval method - Modern SCM + STEP 5: In any pipeline job import the library using @Library directive + @Library('')_ + + // Then call the script by its name (getUser.groovy) + getUser '', '' \ No newline at end of file From faa8e06915b075966a23dc12ae732439f2419fb6 Mon Sep 17 00:00:00 2001 From: jaintpharsha Date: Mon, 9 May 2022 10:33:41 +0530 Subject: [PATCH 15/37] maven 09-May-22 --- maven.txt | 94 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 94 insertions(+) create mode 100644 maven.txt diff --git a/maven.txt b/maven.txt new file mode 100644 index 0000000..33c9ea8 --- /dev/null +++ b/maven.txt @@ -0,0 +1,94 @@ +Maven is a project management tool which can manage complete build lifecycle. +Maven simplifies and standardizes the project build process. + +Build process will be + compile + testing + library dependency + distribution + documentation + deployment + +POM.xml + - project object modal + - In this file we put all the maven configurations. + +Maven folder structure +├── pom.xml +└── src + └── main + ├── java + │   └── com + │   └── yourorganization + │   └── maven_sample + │   └── LogicPositivizer.java + └── resources + └── Blabla.java +└── target + +Maven build life cycle +A build lifecyle is a sequence of task used to build the application. + 1. validate - validate thr project is correct and all the necessary information is + available. + 2. compile - compiles the source code and create obeject code in target folder. + 3. Test - Test the compiled source code using unit testing (JUnit) + 4. Package - All the compiled code is packages in to distribution format which is jar, war + and ear. + 5. integration-test - deplpy the package/build/artifact into an environment where + integration-test can be executed + 6. verify - verify the package is valid and it meets the configured criteria or not. + 7. install - install/copy the package into the local repository. + 8. deploy - If configured, deploy the application/build to integration or relese environment. + +Maven repository + Maven repository is a directory to store all the project jars, libraries, plugins and any artifacts related to project. + 1. Local Repository + - Is a local folder/directory on your machine (where we have installed or running + maven) + - The default location of maven local repository is HOME_DIRECTORY/.m2 directory + - .m2 is created when ever we run mvn command for the first time. + + 2. Central Repository + - Maven central repository is managed by maven community and it contains large number + commonly used libraries. + - We can also publish our own libraries to central repository. + - The default location is https://repo1.maven.org/maven2 + + 3. Remote Repository + - A maven repository setup inside a company or a project related repository but not + public. + - This is a company maintained reposiroty which can be accessed only inside the + company network. + +Maven dependency Search sequence + 1. Search in local repo. + 2. Search in central repo. + 3. Search in remote repo. + + Maven stops searching once it finds the library. + +groupId + Is the unique project name or organization name + - org (organization) + - com (commercial) + - company name + - project/module +artifactId + IS the name of the project / name of build + +versionId + Is the version number of the project + +properties + - To set default encoding type + - To set the maven version + +Difference bertween mvn install vs mvn deploy + install - it will upload the artifact to local repository (.m2 folder) + deploy - uploads the package/artifact to the another/remote repository + +Assignment + what is maven site ? + Check and run Archetype + How to integrate maven with tomcat ? + \ No newline at end of file From 5ef1e310c041763532984e4b27297e86d439c820 Mon Sep 17 00:00:00 2001 From: jaintpharsha Date: Tue, 10 May 2022 11:06:04 +0530 Subject: [PATCH 16/37] Docker image and containers 10-May-22 --- docker.txt | 115 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 115 insertions(+) create mode 100644 docker.txt diff --git a/docker.txt b/docker.txt new file mode 100644 index 0000000..fd456a8 --- /dev/null +++ b/docker.txt @@ -0,0 +1,115 @@ +Docker + Docker is one of the tools that used the idea of the isolated resources to + create a container that allows applications to be packaged with all the + dependencies installed and ran wherever we wanted. + + Docker can only run on Linux machines this means I cant install Dokcer directly on Windows or any other OS. + If I want install Docker on windows then I need to run a Linux VM in windows on top that I need to run Docker. + +Virtualization (VM) + - VM is way of running virtual OS on top a host OS using a special software called Hyperviser. + - VM directly shares the harware of the host OS. + + VM vs Containerisation + 1. Virtualization at hardware level 1. Virtualization at OS level + 2. Heavyweight - consume more host 2. Lightweight + resources + 3. VM useses hypervisor 3. containerisation tool is used + 4. limited performace - Boot up time 4. Native performace - usualy boot + is more which is in minutes fast in seconds. + 5. Cosumes more storage 5. Shres OS storage means only uses + required storage. + 6. Supports all OS 6. Supports on Linux + +host machine + This is the machine in which docker is running + +Docker image + + To list images + docker images + + To download images from docker hub + docker pull : + + note: The default tag will be always latest. + if we wont specify any tag latest will be considered + + To connect to your docker hub account + docker login + + To pull a image from your repo + docoker pull /: + + ex: docker pull harshajain/my_ubuntu + + To push a image to your repo + + 1. Create a tag which matches your repo syntax + docker tag + + ex: docker tag ubuntu:22.10 harshajain/test:1.0 + + 2. Push the image + docoker push /: + + ex: docker push harshajain/test:1.0 + +Docker container + A container is a set of isolated processes and resources. Linux achieves + this by using namespaces, which allows processes to access only resources + in that particular namespace, which allows having a process tree means set + of processes that is completely independent of the rest of the systems processes. + + Docker definition: A container is a standard unit of software that packages + up code and all its dependencies so the application runs quickly and reliably + from one computing environment to another. + + To list running containers + docker ps + (or) + docker container ls + + To list all containers + docker ps -a + + To list all stopped conatainers + docker ps -a --filter status=exited + + To delete one or more container + docker rm .... + + To delete a running container + 1. Forcefully - docker rm -f + 2. Grace fully - docker rm $(docker stop ) + (or) + docker stop | xargs -I{} docker rm "{}" + + To delete all stopped/exited containers + docker rm $(docker ps -aq --filter status=exited) + + + + To check the logs of conatainers + docker logs + + To login / get inside a containre + docker attach + + To run a command inside a conatainer + docker exec -it + + To create a container from a docker image + docker run -it -d --name : + + -it - Interactive Terminal (tty) + -d - deatached mode (when ever we create a container it will auto login to avoid this + we can create a container in detached mode) + --name used to provide user defined conatainer name + + Note: Always use the options before : + + ex: docker run -it -d --name my_jenkins -p 8080:8080 -p 50000:50000 jenkins/jenkins + +Assignment: work with docker commands + Try to create a jenkins container (jenkins/jenkins:lts) \ No newline at end of file From 3fa0e476e182d4feab216fcc9ea0b39f24eed5f0 Mon Sep 17 00:00:00 2001 From: jaintpharsha Date: Tue, 10 May 2022 11:06:43 +0530 Subject: [PATCH 17/37] Docker image and containers 10-May-22 --- docker.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker.txt b/docker.txt index fd456a8..22363a6 100644 --- a/docker.txt +++ b/docker.txt @@ -111,5 +111,5 @@ Docker container ex: docker run -it -d --name my_jenkins -p 8080:8080 -p 50000:50000 jenkins/jenkins -Assignment: work with docker commands +Assignment: work with docker image and container commands Try to create a jenkins container (jenkins/jenkins:lts) \ No newline at end of file From 32ae2d906b32ea4c0edc5a0cd1ffa9dbf72e7cdf Mon Sep 17 00:00:00 2001 From: jaintpharsha Date: Thu, 12 May 2022 10:44:40 +0530 Subject: [PATCH 18/37] Dockerfile 12-May-22 --- docker.txt | 88 +++++++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 81 insertions(+), 7 deletions(-) diff --git a/docker.txt b/docker.txt index 22363a6..df3278c 100644 --- a/docker.txt +++ b/docker.txt @@ -88,17 +88,18 @@ Docker container To delete all stopped/exited containers docker rm $(docker ps -aq --filter status=exited) - + To run a command inside a conatainer + docker exec -it To check the logs of conatainers docker logs To login / get inside a containre - docker attach - - To run a command inside a conatainer - docker exec -it - + docker attach + (safe exit +qp) + (OR) + docker exec -it /bin/bash + To create a container from a docker image docker run -it -d --name : @@ -112,4 +113,77 @@ Docker container ex: docker run -it -d --name my_jenkins -p 8080:8080 -p 50000:50000 jenkins/jenkins Assignment: work with docker image and container commands - Try to create a jenkins container (jenkins/jenkins:lts) \ No newline at end of file + Try to create a jenkins container (jenkins/jenkins:lts) + + docker commit + docker export + docker import + docker save + +Docker custom image / Dockerfile + Dockerfile + Dockerfile is used to create custom images by using any stock image or other image as base image. + In Dockerfile we can write some set of instructions to update any image. + + To create image from Dockerfile + docker build -t my_ubuntu . + + FROM ubuntu + FROM is the first instruction in the every Dockerfile + FROM is used to specify the base image on top which all the other + instruction will run in the same Dockerfile. + + FROM : + + RUN + Normal shell command or the commands supported by the base image are executed using this instruction. + we can have n number of RUN in a single Dockerfile. + + Normal command format + RUN + + exec format + RUN ['','',''] + RUN ['apt','update'] + RUN ['apt','install','-y','git'] + RUN ['ls','-lrt'] + + + ENV + - This instruction is used to set the environment variable inside the container. + + ENV + ENV = + + multiple + ENV = = = .... + + To create environment variables at run time + - using -e or --env option at the runtime we can create env variables + - For multiple variables use multiple -e + + ex: docker run .... -e = -e = .... + + The best way to load multiple env variable is using env file + using --env-file at the runtime (with docker run command) we can + load the env file containing n number variables. + + + COPY and ADD + - Both copy and add instruction is used to copy files and directories from host machine to the image. + - The source path to copy files should always be evaluted with reference to Dockerfile. + + ADD supports extra source formats + - If the source is a compressed file add will automatically + uncompresses it to the destination. + - If the source is a link to a downloadable file it will download + to the destination. + + COPY + ADD + + + + + + \ No newline at end of file From 1e523d55da4cec6cf36cd20ac91e1b6342c2e33d Mon Sep 17 00:00:00 2001 From: jaintpharsha Date: Fri, 13 May 2022 10:01:51 +0530 Subject: [PATCH 19/37] Bind Mounts 13-May-22 --- docker.txt | 45 +++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 41 insertions(+), 4 deletions(-) diff --git a/docker.txt b/docker.txt index df3278c..0213080 100644 --- a/docker.txt +++ b/docker.txt @@ -87,6 +87,8 @@ Docker container To delete all stopped/exited containers docker rm $(docker ps -aq --filter status=exited) + (or) + docker container prune To run a command inside a conatainer docker exec -it @@ -98,8 +100,8 @@ Docker container docker attach (safe exit +qp) (OR) - docker exec -it /bin/bash - + docker exec -it /bin/bash + To create a container from a docker image docker run -it -d --name : @@ -147,7 +149,6 @@ Docker custom image / Dockerfile RUN ['apt','update'] RUN ['apt','install','-y','git'] RUN ['ls','-lrt'] - ENV - This instruction is used to set the environment variable inside the container. @@ -182,7 +183,43 @@ Docker custom image / Dockerfile COPY ADD - + + CMD and ENTRYPOINT + shell format + CMD "ls -lrt" + ENTRYPOINT "ls -lrt" + + EXEC Format + CMD ["ls","-l","-rt"] + ENTRYPOINT ["ls","-lrt"] + + - Both CMD and ENTRYPOINT are used to define the execution command of the container which will be created + from this image. + - If we use multiple CMD or ENTRYPOINT in the same Dockerfile only the latest one will be considered + and all the other CMD or ENTRYPOINT will be ignored. + - If we use both CMD and ENTRYPOINT in the same Dockerfile, ENTRYPOINT will get the + higest priority and the command of CMD will become as argumetns to ENTRYPOINT + + Difference + - CMD command can be overridden at the runtime. + - ENTRYPOINT can't be overridden at the runtime but the runtime command + will become parameters to ENTRYPOINT command. + + Note: Q. Can we override ENTRYPOINT + Yes, after docker 1.6 version docker has given option to over + Entrypoint command at the runtime using --entrypoint + +Docker Volumes + - As the layers inside the image are readonly which means once the image is created + we cannot change/edit so we cannot put the conatainer data in image. + - Container create a top most RW layer and all the runtime data is saved here. + - Container layer is temparary layer, If we loose the container we loose data. so + to retain/persist the container runtime data we need docker volumes. + + Bind Mounts + - we can mount host machine filesystem (files and directories) to the container + + docker run -v : From 561fb3d4dd07d76ae8ef74e04d2a88343dd05ead Mon Sep 17 00:00:00 2001 From: jaintpharsha Date: Sat, 14 May 2022 10:19:19 +0530 Subject: [PATCH 20/37] Volumes, namespaces, expose, publish 14-May-22 --- docker.txt | 73 +++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 72 insertions(+), 1 deletion(-) diff --git a/docker.txt b/docker.txt index 0213080..39e6ae8 100644 --- a/docker.txt +++ b/docker.txt @@ -208,7 +208,14 @@ Docker custom image / Dockerfile Note: Q. Can we override ENTRYPOINT Yes, after docker 1.6 version docker has given option to over Entrypoint command at the runtime using --entrypoint - + + EXPOSE + EXPOSE + - Used to expose a port to the docker network + - This is make the port accessable by all the other containers in the + same docker network. + + Docker Volumes - As the layers inside the image are readonly which means once the image is created we cannot change/edit so we cannot put the conatainer data in image. @@ -221,6 +228,70 @@ Docker Volumes docker run -v : + Docker Volumes + - These are docker managed filesystem and we use docker commands to manage these + volumes + - Volumes are easier to manage, backup or migrate than bind mounts. + - Volumes supports many drivers which means we can maunt many types of filesystem. + - Default location of docker volume is /var/lib/docker/volumes + + docker run -v : + To create volume + docker volume create + + To list volume + docker volume ls + + To Delete volume + docker volume rm + +Namespaces and cgroups + - Docker uses linux namespaces to provide isolated workspace for processes called + conatainer + - when a container is created, docker creates a set of namespaces for it and provides + a layer of isolation for container. + - Each container run in a different namespace + + namespace (To list - lsns) + cgroups + - Linux OS uses cgroups to manage the availale hardware resoruces such as + cpu, RAM, Disk, I/O. + - we can control tje access and also we can apply the limitations + + To list - lscgroup + pid - namespace for managing processes (process isolation) + user - namespace for non-root user on linux. + uts - namespace for unix timesharing system which isolates kernel and version identifiers, + time bonding of process. + ipc - (interprocess communication) namespace for managing the process communication. + mnt - namespace for managing filesystem mounts. + net - namespace for managing network interfaces. + +Docker networking + Publish + PUBLISH = Expose + outside world port mapping + + - publics will bind the container port to the host port which we can access from + outside world using : + + - To publish a port + docker run -p : ..... + + -P publish_all + It binds all the exposed ports of the container to host machine port + + To map direct IP address to the host + port to port + ip:: + ip:: + + Range of ports + many to one + -p 8080-8090:8080 + many to many + -p 8080-8085:8086-8091 + - The total number of host ports in range should be same as the + total number of container ports range. \ No newline at end of file From cefbaad8509a1095ef12c5e4331d7110679019ae Mon Sep 17 00:00:00 2001 From: jaintpharsha Date: Sun, 15 May 2022 10:13:35 +0530 Subject: [PATCH 21/37] Docker networks, multistage, lifecycle, Architecture 15-May-22 --- docker.txt | 148 +++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 144 insertions(+), 4 deletions(-) diff --git a/docker.txt b/docker.txt index 39e6ae8..68a096f 100644 --- a/docker.txt +++ b/docker.txt @@ -168,7 +168,14 @@ Docker custom image / Dockerfile The best way to load multiple env variable is using env file using --env-file at the runtime (with docker run command) we can load the env file containing n number variables. - + + ARG + Using ARG we can pass parameters to Dockerfile as user inputs + + ARG = + + To pass the value at build time + docker build --build-arg = .... COPY and ADD - Both copy and add instruction is used to copy files and directories from host machine to the image. @@ -215,7 +222,12 @@ Docker custom image / Dockerfile - This is make the port accessable by all the other containers in the same docker network. - + WORKDIR + This is used to set the working directory for all the instructions that follows it + such as RUN, CMD, ENTRYPOINT, COPY, ADD + + WORKDIR + Docker Volumes - As the layers inside the image are readonly which means once the image is created we cannot change/edit so we cannot put the conatainer data in image. @@ -253,7 +265,6 @@ Namespaces and cgroups a layer of isolation for container. - Each container run in a different namespace - namespace (To list - lsns) cgroups - Linux OS uses cgroups to manage the availale hardware resoruces such as @@ -294,4 +305,133 @@ Docker networking -p 8080-8085:8086-8091 - The total number of host ports in range should be same as the total number of container ports range. - \ No newline at end of file + +Docker network types + 1. Bridge + - This is a private internal network created by docker on the host machine + by name docker0 + - This is the default network type for all the container which are created + without any network configurations. + - By default all the containers in the same bridge can communicate with + eachother without any extra configuration. + - We cannot use container name for communication only IP address is allowed in + default bridge. + + Custom bridge + To create bridge network + docker network create --driver bridge my_bride + + - In custom bridge containers can communicate with eachother with container + name and also with IP address. + + 2. Host + - This driver removes the network isolation between docker and the host. + - The containers are directly connected to host machine network without + extra layer of any docker network. + - Shares the same TCP.IP stack and same namespace of host machine. + - All the network interfaces which are there in host machine are + accessable by this container. + + 3. None + - Containers are not attached to any network by docker. + - All the required network configurations need to be done + manually. + - The host or any other containers won't be able to communicate + with this container untill a custom network is configured. + +ocker Architecture + Docker Daemon + - A background process that manages docker images, containers, network and volumes. + - This Daemon constantly listens for docker API request and processes them. + + Docker REST API + - API which is used by applictions to interact with the docker daemon. + + Docker CLI + - It is a command line interface for interacting with docker daemon through + REST api. + + Docker Objects + +Benifits of Docker + Flexible: + Complex applictions cab be divided and containerised in small compenets called + microservice. + + Lightweight: + Containers share the machine’s OS system kernel and therefore do not require + an OS per application, driving higher server efficiencies and reducing server and licensing costs + + portable: + we can build images anywhere and then deploy to cloud, run anywhere. + +States of conatiner / Lifecycle of container + 1. Created - if container is newly created and container is not yet started. + 2. Running - A currently running container. It means there is no problem + with container to run the process. + 3. Exited - A container ran and completed ot executiom with failure. + 4. paused - A container whose process have been paused. (we can unpause the container) + 5. Dead - if docker daemon tried and failed to stop a container (host ram full) + 6. Restarting - container will be in the phase of retarting the main process. + +Multistage build + How to optimize docker build process ? + How to reduce the size of the docker image or cotainer ? + + After docker 1.6 version docker released this option. + 1. There are 2 problems with the normal build process + 1. Size: challenge is to keep the image and its containers size as minimal as possible. + 2. larger the surface area more the application is vurnalable to attacks. + + - Multistage build allows us to define multiple FROM in same Dockerfile. + - Dependency between multile FROM is maintained by naming FROM using + AS keyword and we can refer this name in another FROM. + + FROM AS + + - Only the final FROM image is created leaving back all the other FROM. + - Copy only the required files from the named FROM stage like below. + FROM final_build + COPY --from + + 2. Always try to use the slim / alpine / stretch version of base image instead + od using the fully loaded base image. + + exapmle: https://github.com/jaintpharsha/docker_multi_build.git + +docker-compose + Installation + 1. sudo curl -L "https://github.com/docker/compose/releases/download/1.29.2/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose + 2. sudo chmod +x /usr/local/bin/docker-compose + 3. sudo ln -s /usr/local/bin/docker-compose /usr/bin/docker-compose + + - docker-compose is a tool for defining and running multile container docker application with + a single command. + - We use YAML file to do docker related configurations then with a single command + we can execute this YAML file to create docker objects defined in this file. + + docker-compose.yml + version: "3.8" + services: + jenkins: + image: jenkins/jenkins:lts + container_name: dc-jenkins + ports: + - "8080:8080" + - "5000:5000" + networks: + - my_brid + alpine: + build: . + container_name: dc-ubuntu + volumes: + - my_vol:/home + networks: + - my_brid + networks: + my_brid: + driver: bridge + + volumes: + my_vol: {} + \ No newline at end of file From 8aba4d5a1b0c30ba17a76b6ce33502856bdb4077 Mon Sep 17 00:00:00 2001 From: jaintpharsha Date: Tue, 17 May 2022 10:55:30 +0530 Subject: [PATCH 22/37] k8s setup 17-May-22 --- kubernetes.txt | 67 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) create mode 100644 kubernetes.txt diff --git a/kubernetes.txt b/kubernetes.txt new file mode 100644 index 0000000..55d3e38 --- /dev/null +++ b/kubernetes.txt @@ -0,0 +1,67 @@ +Kubernetes Installation + Minimum requirement for K8S master node is (2-core CPU and 2GB Ram) + + 1. sudo apt update + 2. sudo apt-get install -y apt-transport-https + 3. sudo su - + 4. curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add + 5. echo 'deb http://apt.kubernetes.io/ kubernetes-xenial main' > /etc/apt/sources.list.d/kubernetes.list + + 6. exit from sudo + 7. sudo apt update + 8. sudo apt install -y docker.io + + 9. sudo systemctl start docker + 10. sudo systemctl enable docker.service + + 11. sudo apt-get install -y kubelet kubeadm kubectl kubernetes-cni + + Take ami from the above ec2 instances to create worker nodes + + 12. Login back to master node, make sure below steps are executed before running kubeadm init + 1. sudo su - + 2. docker cgroup driver configuration need to be updated + 1. add the below content to the file /etc/docker/daemon.json + { + "exec-opts": ["native.cgroupdriver=systemd"] + } + 2. systemctl daemon-reload + systemctl restart docker + systemctl restart kubectl + + 13. kubeadm init + if this command executes successfully then we get kubeadm join command with token + save this command in seperate file for worker nodes to add to this master. + + 14. k8s configurations for kubectl command + 1. exit from root + 2. copy the default k8s conf file to home + a. mkdir -p $HOME/.kube + b. sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config + c. sudo chown $(id -u):$(id -g) $HOME/.kube/config + + 15. Now Install k8s CNI driver + 1. sudo sysctl net.bridge.bridge-nf-call-iptables=1 + 2. kubectl apply -f "https://cloud.weave.works/k8s/v1.13/net.yaml" + + + 16. Login to worker nodes + a. sudo su - + + b. update docker cgroup driver configuration need to be updated + 1. add the below content to the file /etc/docker/daemon.json + { + "exec-opts": ["native.cgroupdriver=systemd"] + } + 2. systemctl daemon-reload + systemctl restart docker + systemctl restart kubectl + + 17. For now open all ports in master + + 18. run the kubeadm join command which we get from kubeadm inint from master + + 19. In master node check for the worker nodes. + kubectl get nodes + + \ No newline at end of file From 94268582f6c04e923db86a4d535d3184c309d166 Mon Sep 17 00:00:00 2001 From: jaintpharsha Date: Thu, 19 May 2022 10:10:02 +0530 Subject: [PATCH 23/37] kubernetes Architecture 19-May-22 --- kubernetes.txt | 65 +++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 62 insertions(+), 3 deletions(-) diff --git a/kubernetes.txt b/kubernetes.txt index 55d3e38..e51d92e 100644 --- a/kubernetes.txt +++ b/kubernetes.txt @@ -43,8 +43,7 @@ Kubernetes Installation 15. Now Install k8s CNI driver 1. sudo sysctl net.bridge.bridge-nf-call-iptables=1 2. kubectl apply -f "https://cloud.weave.works/k8s/v1.13/net.yaml" - - + 16. Login to worker nodes a. sudo su - @@ -64,4 +63,64 @@ Kubernetes Installation 19. In master node check for the worker nodes. kubectl get nodes - \ No newline at end of file +kubernetes Architecture + The architecture of k8s differs from master and worker node + + Master node components + 1. Api Server / kube-api-server + - It is the main managemnet point of the cluster and also called + as brain of the cluster. + - All the components are directly connected to API serve, they + communicate through API server only and no other component will + communicate directly with eachother. + - This is the only component which connects and got access to etcd. + - All the cluster requests are authenticated and authorized by API server. + - API server has a watch machanism for watching the changes in cluster. + + 2. etcd + - ectd is a distributed , consistent key value store used for + storing the complete cluster information/data. + - ectd contains data such as configuration managemnet of cluster, + distributed work and basically complete cluster information. + + 3. scheduler / kube-scheduler + - The scheduler always watches for a new pod request and + decides which worker node this pod should be created. + - Based on the worker node load, affinity and anti-affiny, taint configuration + pod will be scheduled to a particualr node. + + Controller manager /control manager / kube-controller + - It is a daemon that always runs and embeds core control loops known as controllers. + - K8s has some inbuild controllers such as Deployment, DaemonSet, ReplicaSet, Replication controller, + node controller, jobs, cronjob, endpoint controller, namespace controller etc. + + Cloud controller manager + + + + Worker node components + kubelet + - It is an agent that runs on each and every worker node and it alsways watches the API + server for pod related changes running in its worker node. + - kubelet always make sure that the assigend pods to its worker node is running. + - kubelet is the one which communicates with containarisation tool (docker daemon) + through docker API (CRI). + - work of kubelet is to create and run the pods. Always reports the status of the worker node + and each pod to API server. (uses a tool call cAdvisor) + - Kubelet is the one which runs probes. + + kube service proxy + (in k8s service means networking) + - Service proxy runs on each and every worker node and is responsble for watching API + server for any changes in service configuration (any network related configuration). + - Based on the configuration service proxy manages the entire network of worker node. + + Container runtime interface (CRI) + - This component initialy identifies the container technology and connects it to kubelet. + + + pod + - pods are the smallest deployable object in kuberntes. + - pod should contain atleast one container and can have n number of containers. + - If pod contains more than one container all the container share the same memory assigned to that pod. + \ No newline at end of file From 5df397214c725d6d63c5c2b05eab18543593bdef Mon Sep 17 00:00:00 2001 From: jaintpharsha Date: Fri, 20 May 2022 09:28:10 +0530 Subject: [PATCH 24/37] YAML syntax 20-May-22 --- kubernetes.txt | 64 +++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 63 insertions(+), 1 deletion(-) diff --git a/kubernetes.txt b/kubernetes.txt index e51d92e..2060a9b 100644 --- a/kubernetes.txt +++ b/kubernetes.txt @@ -123,4 +123,66 @@ kubernetes Architecture - pods are the smallest deployable object in kuberntes. - pod should contain atleast one container and can have n number of containers. - If pod contains more than one container all the container share the same memory assigned to that pod. - \ No newline at end of file + +YAML file + - filetype .yaml or .yml + - YAML file contains key - value pairs where key are fixed and defined by the + kubernetes and value is user defined configuration. + - Values supoorts multiple datatypes - string, Integer, Boolean, Array, List. + + example: List representation + + 1) name: Harsha + hobbies: ["Driving","coding"] + + (or) + + name: Harsha + hobbies: + - Driving + - coding + +example pod + + apiVersion: v1 + - This is the version of api used to create a k8s object. + - The fields are casesensitive and YAML use camelcase. + - The type of api are alpha, beta and stable. + + kind: Pod + - here we specify which object we need to create. + - Always object name first letter is capital. + + metadata: + - This field is used to provide information on the object + which we are creating. + - Information such as name, labels and annotaions. + + spec: + - This is used to do the actual configuration of the + object. + +apiVersion: v1 +kind: Pod +metadata: + name: my-first-pod +spec: + containers: + - name: my-nginx + image: nginx:latest + ports: + - containerPort: 80 + +TO create / apply a configuration + kubectl apply -f .yml + +To list objects + kubectl get + ex: List pods - kubectl get pods + List deployment - kubectl get deployments + +To delete objects + kubectl delete + + +Assignment: What happens if we create a pod with kubectl ? \ No newline at end of file From a1fe3fa1682aed9d134f1cccb7a6912e704c26b2 Mon Sep 17 00:00:00 2001 From: jaintpharsha Date: Sun, 22 May 2022 10:34:39 +0530 Subject: [PATCH 25/37] selectors labesl annotations deployment daemonset 22-May-22 --- kubernetes.txt | 124 ++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 123 insertions(+), 1 deletion(-) diff --git a/kubernetes.txt b/kubernetes.txt index 2060a9b..5b121fd 100644 --- a/kubernetes.txt +++ b/kubernetes.txt @@ -185,4 +185,126 @@ To delete objects kubectl delete -Assignment: What happens if we create a pod with kubectl ? \ No newline at end of file +Assignment: What happens if we create a pod with kubectl ? + +K8S Labels and selectors + - K8S labels is a metadata key value which can be applied to any object in k8s. + - Labels are used to identify by using selectors. + - Multiple objects can have same label, multiple labels to same object and Label lenght should be less that 63 characters. + + TO list all labels of a object + kubectl get --show-labels + + + Selectors + - Selectors are used to filter and identifly the labeled k8s object. + + Equality-Based + - It will use only one label in comparision and it will look for objects with exact same + string in label. + - we can use 3 types of operators equal ( = or == ) and not-qual ( != ) + + example: + selectors: + matchLabels: + app=nginx + (or) + app: nginx + + set-based + - This type of selector allows us to filter objects based on multiple set of values to a label key. + - 3 types of operators we can use in, notin and exists. + + example: + selectors: + matchLabels: + app in (nginx, my-nginx) + app exits (nginx, my-nginx) + app notin (nginx, my-nginx) +Annotations + - These are used for record purpose only and to provide some user information to objects. + - These are non-identifying metadata so we cannot use selectors on annotations. + + example: personal_info, phone_number, imageregistry, author + +Assignment: Difference b/w set-based and equality-based selector. + Difference b/w labels and annotations. + +ReplicaSet vs Replication Controller + - Both ensures that a specified number of pod replicas are alyways running at a given point of time. + - Replication controller is a very old way of replicating the pos and now it is replaced by ReplicaSet + + - The only differenece b/w them is their selector types. + Replication Controller supports only equality-based selector. + ReplicaSet supports both equality-based and set-based selectors. + +Deployment controller / Deployment / k8s deployment + - Deployment is used to create replicas of pod and it makes sure at a given point of time + the number of replicas of pod is alway running. + - Deployment internally uses ReplicaSet to replicate the pods. + - If we update the configuration in deployment it will automatically updates it to all the pods. + - Rollout and Rollback of pod update is possible. + - we can pause a deployment whenerver we need. + - Deployment has got its own internal autoscaller which is of type horizontal scaller. + To apply calling + kubectl autoscale deployment.v1.apps/ --min=5 --max=20 --cpu-percent=50 + - scaleup and scaledown is possible by increasing and decreasing the replica count at any given + point of time. + kubectl scale deployment.v1.apps/ --replicas=10 + + - deployment is a cluster level object. + + deployment = pod + ReplicaSet + autoscaling + RollingUpdates + + Deployment spec file. + + apiVersion: apps/v1 + kind: Deployment + metadata: + name: nginx-deployment-new + labels: + app: my-deployment-nginx + spec: + replicas: 5 + selector: + matchLabels: + app=nginx + template: + metadata: + labels: + app: nginx + spec: + containers: + - name: nginx + image: nginx:1.14.2 + ports: + - containerPort: 80 + +Assignment: demo on selectors types + +DaemonSet + - DaemonSet ensures that a copy of pod is always running on all the worker nodes in the cluster. + -If a new node is added or if deleted DaemonSet will automatically adds/deletes the pod. + + usage: + - we use DaemonSet to deploy monitoring agents in every worker node. + - Log collection daemons: to grab the logs from worker and all the pods running in it. + +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: nginx-daemonset +spec: + selector: + matchLabels: + app: daemonset-nginx + template: + metadata: + labels: + app: daemonset-nginx + spec: + containers: + - name: nginx + image: nginx:1.14.2 + ports: + - containerPort: 80 \ No newline at end of file From 1da038b72ec1dda85e19917fdc2a0f9975ddd270 Mon Sep 17 00:00:00 2001 From: jaintpharsha Date: Tue, 24 May 2022 10:31:09 +0530 Subject: [PATCH 26/37] stateful/stateless StatefulSet monolothic/microservices service-NP,CI 24-May-22 --- kubernetes.txt | 94 +++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 93 insertions(+), 1 deletion(-) diff --git a/kubernetes.txt b/kubernetes.txt index 5b121fd..034894b 100644 --- a/kubernetes.txt +++ b/kubernetes.txt @@ -307,4 +307,96 @@ spec: - name: nginx image: nginx:1.14.2 ports: - - containerPort: 80 \ No newline at end of file + - containerPort: 80 + +Statefull Applications + - User session data is saved at the server side. + - if server goes down, it is difficult to transfer the session data to other server. + - This type of application will not work, if we want to implement autoscalling. + +Stateless Applications + - user sessiondata is never saved at the server side. + - using a common authentication gateway / client token method to validate the users + once for multiple microservices. + +https://medium.com/tech-tajawal/microservice-authentication-and-authorization-solutions-e0e5e74b248a + +Monolothic and Microservice architecture + + Monolothic architecture + - A monolothic application has a single code base with multiple moduels in it. + - It is a single build for entire application. + - To make minor changes to application, we need to re-build and re-deploy the + complete application. + - scalling is very challenging. + + Microservice architecture + - A microservice application is composed of small (micro) services. + - Each service will have a different code base. + - Application are divided into as small as possible sub applications called services + which are independent to each other which are called loosly coupled. + - Each service can be managed seperately and it is deployable seperately. + - Services need not to share same technology stack or frameworks. + +StatefulSet + - StatefulSet = Deployment + sticky identity for each and every pod replica. + - Unlike a deployment a StatefulSet maintains a sticky identity for each of the pod. + +Node controller + - Looks for node statuses and reponds to API server only when a node is down. + +Endpoint Controller + - Populates the information of endpoints of all the objects. + +Service (svc) + - Service is an REST api objects with set of policies for defining the + access to set of pods. + - Services are the default load balancer in k8s. + - services are always created and works at cluster level. + - services are the networking configurations which we do in k8s. + - k8s prefers to use 30000 - 50000 range of ports to define services. + +1. ClusterIP + - This is the default type of service which exposes the IPs of pod to the other pods + with in the same cluster. + - ClusterIP cannot be accessed outside cluster. + - services are the default loadbalancers of k8s. + + apiVersion: v1 + kind: Service + metadata: + name: my-svc + spec: + type: ClusterIP + selector: + app: my-nginx + ports: + - name: http + port: 30080 + targetPort: 8080 + +2. nodePort + - A nodeport service is the most primitive way to get the external traffic directed to our services / applications + running inside a pod within the cluster. + - By default NodePort acts as a load balancer. + - Automatically a ClusterIP will be created internally. + + NodePort = ClusterIP + a port mapping to the all the nodes of cluster. + + - If we wont specify any port while creating nodeport, k8s will automatically asign a port between the range 30000 - 32767 + - By default nodeport will open the port in all the node in cluster including master node. + + apiVersion: v1 + kind: Service + metadata: + name: my-svc + spec: + type: NodePort + selector: + app: my-nginx + ports: + - name: http + nodePort:30082 + port: 8080 + targetPort: 80 + \ No newline at end of file From 95e96544860f72a628f7cad923961abb7f9cab12 Mon Sep 17 00:00:00 2001 From: jaintpharsha Date: Wed, 25 May 2022 10:25:38 +0530 Subject: [PATCH 27/37] namespaces 25-May-22 --- app/Dockerfile | 6 ++++ app/main.py | 18 ++++++++++ app/requirements.txt | 1 + kubernetes.txt | 85 +++++++++++++++++++++++++++++++++++++++++++- 4 files changed, 109 insertions(+), 1 deletion(-) create mode 100644 app/Dockerfile create mode 100644 app/main.py create mode 100644 app/requirements.txt diff --git a/app/Dockerfile b/app/Dockerfile new file mode 100644 index 0000000..6c71c97 --- /dev/null +++ b/app/Dockerfile @@ -0,0 +1,6 @@ +FROM python +WORKDIR /app +COPY main.py /app +COPY requirements.txt /app +RUN pip install -r requirements.txt +ENTRYPOINT python main.py diff --git a/app/main.py b/app/main.py new file mode 100644 index 0000000..26dbe5f --- /dev/null +++ b/app/main.py @@ -0,0 +1,18 @@ +from flask import Flask +import socket, os + +app = Flask(__name__) + +@app.route('/') +def print_ip(): + hostname = socket.gethostname() + local_ip = socket.gethostbyname(hostname) + return local_ip + # return "This is homepage or root /" + +@app.route('/login') +def print_login(): + return "This is login page or path /login" + +if __name__ == "__main__": + app.run(host="0.0.0.0", port=80) diff --git a/app/requirements.txt b/app/requirements.txt new file mode 100644 index 0000000..8ab6294 --- /dev/null +++ b/app/requirements.txt @@ -0,0 +1 @@ +flask \ No newline at end of file diff --git a/kubernetes.txt b/kubernetes.txt index 034894b..5949ac0 100644 --- a/kubernetes.txt +++ b/kubernetes.txt @@ -399,4 +399,87 @@ Service (svc) nodePort:30082 port: 8080 targetPort: 80 - \ No newline at end of file + +3. Load Balancer + - It is a tyoe of service which is used to link exernatl load balancer to the cluster. + - This type of serviec is used by cloud providers and this service is completely depends on cloud providers. + - K8s now proides a better alternative for this service type which is called Ingress. + +Assignment: + +How to use custom images / connect to a registry through k8s + 1. Login to the docker hub account + docker login + 2. Create a app to print ip + using flask + 3. push the image to your registry + 4 use the above custom image in k8s spec file. + + image: /: + imagePullPolicy: IfNotPresent + + 5. Create a service of type NodePort attaching the above pods + +namespaces + - k8s namespaces is a way of applying abstraction / isolation to support multiple + virtual clusters of k8s objects with in the same physical cluster. + - Each and every object in k8s must be in a namespac. + - If we wont specify namespace, objects will be created in default namespace of k8s. + - namespaces are cluster level. + - by default pods in same namespace can communicate with eachother. + - Namespace are only hidden from eachother but not fully isolated because one + service in a namespace can talk to another service in another namespace using + fullname (service/) followed by namespace name + + usage: we can apply environment based logical separation on cluster. + + Type of deafault NS + 1. default + - This NS is used for all the objects which are not belongs to any other namespace. + - If we wont specify any namespace while creating an object in k8s then + that object will be created in deafult namespace. + + 2. kube-system + - This namespace is always used for objects created by the k8s system. + + 3. kube-public + - The objects in this namespace are available or accessable to all. + - All the objects in this namespace are made public. + + 4. kube-node-lease + - This namespace holds lease objects assosiated with each node. + - Node lease allows the kubelet to send heartbeats so that the control palne can + detect node failure. + + To list namespace + kubectl get namespaces + + To list objects in a namespace + kubectl get pods --namepsace + (OR) + kubectl get pods -n + + To list obects from all namespaces + kubectl get pods --all-namespaces + + To create a namespace + kubectl create namespace + + To create k8s object in a namespace + 1. in the spec file + metadata: + namespace: + + 2. Using the apply command + kubectl apply -n -f .yml + + Note: what if we use both inside specfile and also in apply command + - apply command check and compares the namespace and wont allow to create the obejct if the namespace is different. + +Assignment: try exec to a pod + + - pod to pod communication is open if the 2 pods are in the same namespace. + - If the pods are in different namespace by default they can't communicate we need a service object for this. + + How a microservice will communicate with other microservice + What is service discovery in k8s \ No newline at end of file From 982a23760137812e436c37201266bfa94e83aa12 Mon Sep 17 00:00:00 2001 From: jaintpharsha Date: Thu, 26 May 2022 12:59:43 +0530 Subject: [PATCH 28/37] headless service 26-May-22 --- kubernetes.txt | 73 ++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 71 insertions(+), 2 deletions(-) diff --git a/kubernetes.txt b/kubernetes.txt index 5949ac0..75dd4c5 100644 --- a/kubernetes.txt +++ b/kubernetes.txt @@ -481,5 +481,74 @@ Assignment: try exec to a pod - pod to pod communication is open if the 2 pods are in the same namespace. - If the pods are in different namespace by default they can't communicate we need a service object for this. - How a microservice will communicate with other microservice - What is service discovery in k8s \ No newline at end of file + +How a microservice will communicate with other microservice +What is service discovery in k8s + +Service discovery + There are 2 ways of dicovering a service in k8s + + 1. Services + we can use the full name of service to discovery a microservice (pod). + service/ + + 2. DNS + - DNS server is added to the cluster in order to watch the k8s service request. + - API server will create DNS records for each new service. + - Record A type is used in k8s service discovery and this DNS is created on service + and pod objects. + + syntax of k8s DNS + ... + + ex: np-ip-app.default.svc.cluster.local + + 3. ENV variables + - which ever the pod that runs on a node, k8s adds environment variables for + each of them to identify the service running in it. + + https://dev.to/narasimha1997/communication-between-microservices-in-a-kubernetes-cluster-1n41 + +Headless service + headless service + - When we neither need nor want loadbalancig and a single IP point to a service, we need use headless service. + - Headless service returns all the ips of the pods it is selecting. + - headless serivce is created by specifing none for clusterIP + - headless service is usually used with statefulsets. + + headless with in cluster + + apiVersion: v1 + kind: Service + metadata: + name: my-svc + spec: + clusterIP: None + selector: + app: my-nginx + ports: + - name: http + port: 30080 + targetPort: 8080 + + headless with nodeport + nodePort = headless + port mapping + + apiVersion: v1 + kind: Service + metadata: + name: my-svc + spec: + clusterIP: None + type: NodePort + selector: + app: my-nginx + ports: + - name: http + nodePort:30082 + port: 8080 + targetPort: 80 + + 1. Create a headless service with statefulsets + 2. Login to any one of pod + 3. apt install dnsutils and do nslookup \ No newline at end of file From 163ab11927c0b280699f97e4bc88b703909f72af Mon Sep 17 00:00:00 2001 From: jaintpharsha Date: Fri, 27 May 2022 10:05:36 +0530 Subject: [PATCH 29/37] pv, pvc, pod status and lifecycle 27-May-22 --- kubernetes.txt | 115 ++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 114 insertions(+), 1 deletion(-) diff --git a/kubernetes.txt b/kubernetes.txt index 75dd4c5..0bcaa12 100644 --- a/kubernetes.txt +++ b/kubernetes.txt @@ -551,4 +551,117 @@ Headless service 1. Create a headless service with statefulsets 2. Login to any one of pod - 3. apt install dnsutils and do nslookup \ No newline at end of file + 3. apt install dnsutils and do nslookup + +pod phases / status / states / life cycle + 1. pending + - This is the status of pod when pod will be waiting for k8s cluster to accept it. + - pod will be downloading the image from registry. + - pod wiil be in pending till the scheduler assigns a node to the pod. + + 2. Running + - The pod has been assigned a node and all the containers inside the pod is running. + - Atleast one container is in running state and others in starting or restarting state then pod will show + running state. + + 3. Failed + - All the container in the pod should not be running and any one container being terminated in failure. + + 4. Succeeded + - ALl the containers in pod have been terminated successfully/gracefully. + + 5. Unknown + - For some reason the state of the pod could not be obtaied by API server. + - The status may occur when k8s cannot communicate with the kubelet or the worker node. + + +terminating + - when pod is being deleted. + +container status + Running + - Means container is running the process inside without any error + Terminated + - Process inside the container has completed the execution or may be failed due to some error. + waiting + - If a container is not running or neither in terminated state. + +Common errors + ImagePullBackOff + - Docker image registry is not accessible. + - Image name / tag version specified is incorrent. + CrashLoopBackOff + - We get this error when probe check has failed. + - Docker image may be faulty. + RunContainerError + - Configmap / secrets are missing. + - Volumes are not available + + +k8s volumes + persistent volume (pv) + - It is a storage space which can be claimend to any pod in the cluster. + - These are cluster level object and not bound to namespace. + + we can control the access to volume in 3 ways: + - ReadOnlyMany(ROX) allows being mounted by multiple nodes in read-only mode. + - ReadWriteOnce(RWO) allows being mounted by a single node in read-write mode. + - ReadWriteMany(RWX) allows multiple nodes to be mounted in read-write mode. + + Note: If we need write access to volume from multiple pods scheduled in mulitple nodes then use ReadWrtieMany + + apiVersion: v1 + kind: PersistentVolume + metadata: + name: my-pv + labels: + volume: test + spec: + storageClassName: local + accessModes: + - ReadWriteOnce + capacity: + storage: 2Gi + hostPath: + path: "/home/ubuntu/my-pv-volume" + + Persistent volume claim (pvc) + - This is the object used to claim / mount the required amount of storage from persistent volume to any + pod in the cluster. + - After we create the PersistentVolumeClaim, the Kubernetes control plane looks for a PersistentVolume that + satisfies the claim's requirements. + - If the control plane finds a suitable PersistentVolume with the same StorageClass, it binds the claim to the volume. + + apiVersion: v1 + kind: PersistentVolumeClaim + metadata: + name: my-pvc + spec: + storageClassName: local + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + + using this in a pod + + apiVersion: v1 + kind: Pod + metadata: + name: my-pvc-pod + spec: + volumes: + - name: pvc-volume + persistentVolumeClaim: + claimName: my-pvc # This name should be same the PVC object name + containers: + - name: my-nginx + image: nginx:latest + ports: + - containerPort: 80 + volumeMounts: + - mountPath: "/usr/share/nginx/html" + name: pvc-volume # This name should be same as the above volume name + + \ No newline at end of file From ac077f8cb9b48f66489a9f094585061f8d5f4dde Mon Sep 17 00:00:00 2001 From: jaintpharsha Date: Sat, 28 May 2022 10:47:58 +0530 Subject: [PATCH 30/37] pod patterns / container types 28-May-22 --- kubernetes.txt | 102 ++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 101 insertions(+), 1 deletion(-) diff --git a/kubernetes.txt b/kubernetes.txt index 0bcaa12..c7b126c 100644 --- a/kubernetes.txt +++ b/kubernetes.txt @@ -664,4 +664,104 @@ k8s volumes - mountPath: "/usr/share/nginx/html" name: pvc-volume # This name should be same as the above volume name - \ No newline at end of file +init container + - init containers are the containers that will run completely before starting + the main app container. + - This provides a lifecycle at the startup and we can define things for + initialization purpose. + - kubernetes has stopped support of probes in init containers. + - These are pod level objects. + - we can use this container to have some deply on the startup of the main container. + + + These are some of the scenarios where you can use this pattern + - You can use this pattern where your application or main containers need some + prerequisites such as installing some software, database setup, permissions on the file + system before starting. + - You can use this pattern where you want to delay the start of the main containers. + + apiVersion: v1 + kind: Pod + metadata: + name: init-container + labels: + app: init-app + spec: + containers: + - name: nginx + image: nginx:1.14.2 + ports: + - containerPort: 80 + volumeMounts: + - mountPath: "/usr/share/nginx/html" + name: workdir + initContainers: + - name: busybox + image: busybox + command: ["/bin/sh"] + args: ["-c","echo '

I am init container new version

' >> /work-dir/index.html"] + volumeMounts: + - mountPath: "/work-dir" + name: workdir + dnsPolicy: Default + volumes: + - name: workdir + emptyDir: {} + + 1. login to pod + kubectl exec -it -- /bin/sh + 2. apt update && apt install -y curl + 3. curl localhost + + To check the log of particular container out of multiple in a pod + kubectl logs -c + +sidecat container + - These are the containers that will run along with the main app container. + - we have a app conaitner which is working fine but we want to extend the + functionality without changing the existing code in main container for this + purpose we can use sidecar container. + - we use this container to feed the log data to monitoring tools. + + These are some of the scenarios where you can use this pattern + - Whenever you want to extend the functionality of the existing single container pod without + touching the existing one. + - Whenever you want to enhance the functionality of the existing single container pod + without touching the existing one. + - You can use this pattern to synchronize the main container code with the git server pull. + - You can use this pattern for sending log events to the external server. + - You can use this pattern for network-related tasks. + + apiVersion: v1 + kind: Pod + metadata: + name: sidecar-container + labels: + app: adaptor-app + spec: + containers: + - name: nginx + image: nginx:1.14.2 + ports: + - containerPort: 80 + volumeMounts: + - mountPath: "/var/log/nginx" + name: logs-dir + - name: side-car + image: busybox + command: ["/bin/sh"] + args: ["-c","while true; do cat /var/log/nginx/access.log /var/log/nginx/error.log; sleep 10; done"] + volumeMounts: + - mountPath: "/var/log/nginx" + name: logs-dir + dnsPolicy: Default + volumes: + - name: logs-dir + emptyDir: {} + + + Adaptor container + - In this patter we use a sidecar container to feed the log data to a monotoring + tool. + + https://www.magalix.com/blog/kubernetes-patterns-the-ambassador-pattern \ No newline at end of file From e1d8aea34ed5805cfe05b8ce181024381ba310ba Mon Sep 17 00:00:00 2001 From: jaintpharsha Date: Mon, 30 May 2022 09:59:14 +0530 Subject: [PATCH 31/37] probes 30-May-22 --- app/main.py | 6 +++ kubernetes.txt | 129 ++++++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 134 insertions(+), 1 deletion(-) diff --git a/app/main.py b/app/main.py index 26dbe5f..ec5e281 100644 --- a/app/main.py +++ b/app/main.py @@ -1,4 +1,5 @@ from flask import Flask +from flask import jsonify import socket, os app = Flask(__name__) @@ -14,5 +15,10 @@ def print_ip(): def print_login(): return "This is login page or path /login" +@app.route('/any_path') +def print_login(): + resp = jsonify(success=True) + return resp + if __name__ == "__main__": app.run(host="0.0.0.0", port=80) diff --git a/kubernetes.txt b/kubernetes.txt index c7b126c..bd0c228 100644 --- a/kubernetes.txt +++ b/kubernetes.txt @@ -764,4 +764,131 @@ sidecat container - In this patter we use a sidecar container to feed the log data to a monotoring tool. - https://www.magalix.com/blog/kubernetes-patterns-the-ambassador-pattern \ No newline at end of file + https://www.magalix.com/blog/kubernetes-patterns-the-ambassador-pattern + +Probes +- probe is a periodic call to some applciation endpoints within a container. + - probes can track success or failure of the other applications. + - When there is a subsequent failure occures we can defie probe to get triggered. + - when subsequent success after a failure we can define probe to get triggered. + - probes works at container level. + + Common fields in probes + initialDelaySeconds + - After the container has started the number of seconds to wait before the probe os triggered. + periodSeconds + - The number of seconds interval the probe should be executed. (Default 10 seconds and minimum 1 second) + timeoutSeconds + - Number of seconds after which probe timeouts. (default 1) + + failureThreshold + - When a probe fails this is the number of subsequent fail times the probe checks the status of application. + - After the number of subsequent failure then probe fails. + - Default value 3 with minimum value 1 + + successThreshold + - minimum number of subsequent success for a probe. + - Default value is 1 + + Endpoints + http probes (httpGet) + host - hostname to conenct and probe will check the status of this hostname + - Default is the IP of current pod + ex: www.google.com + path - exact path to access the application on the http server + ex: /gmail + httpHeaders + - can send custom header messages with the request. + port + - Name or number of the port to access the application + + TCP probes + port + - Name or number of the port to access the application + + exec + commad + - we execute a command and check its status. + +Liveness probe + - The livenessprobe is used to determine if the applciation inside the container + is healthy or needs to be restarted. + - If livenessprobe fails it will mark the container to be retarted by kubelet. + + 1. LivenessPRobe with http + + apiVersion: v1 + kind: Pod + metadata: + name: liveness-http + spec: + containers: + - name: liveness + image: k8s.gcr.io/liveness + args: + - /server + livenessProbe: + httpGet: + path: /healthz + port: 8080 + initialDelaySeconds: 3 + periodSeconds: 3 + + 2. TCP + livenessProbe: + tcpSocket: + port: 8080 + initialDelaySeconds: 3 + periodSeconds: 3 + 3. exec + livenessProbe: + exec: + command: ["",""] + initialDelaySeconds: 3 + periodSeconds: 3 + + 4. named port + ports: + - name: liveness-port + containerPort: 8080 + hostPort: 8080 + livenessProbe: + httpGet: + path: /healthz + port: liveness-port + initialDelaySeconds: 3 + periodSeconds: 3 + +Readiness Probe + - ReadinessProbe is used to determine that a application running inside a + container is in a state to accept the traffic. + - When this probe is successful, the traffic from the loadbalancer is allowed + to the application inside the conatiner. + - When this probe is fails, the traffic from the loadbalancer is halted + to the application inside the conatiner. + readinessProbe: + tcpSocket: + port: 8080 + initialDelaySeconds: 15 + periodSeconds: 10 + +Startup Probe + - This probe will run at the initial start of the container. + - This probe allows us to give maximum startup time for application before + running livenessProbe or readinessprobe. + + startupProbe: + httpGet: + path: /healtz + port: 8080 + initialDelaySeconds: 3 + periodSeconds: 3 + +Assigment: create livenessProbe and readinessprobe with tcp, http, exec + + + + + + + \ No newline at end of file From b0408efbefa172f80075d616a5193290b7446366 Mon Sep 17 00:00:00 2001 From: jaintpharsha Date: Fri, 3 Jun 2022 10:04:22 +0530 Subject: [PATCH 32/37] configmaps 03-June-22 --- kubernetes.txt | 134 ++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 132 insertions(+), 2 deletions(-) diff --git a/kubernetes.txt b/kubernetes.txt index bd0c228..e3a4e90 100644 --- a/kubernetes.txt +++ b/kubernetes.txt @@ -499,7 +499,7 @@ Service discovery and pod objects. syntax of k8s DNS - ... + ...cluster.local ex: np-ip-app.default.svc.cluster.local @@ -885,10 +885,140 @@ Startup Probe periodSeconds: 3 Assigment: create livenessProbe and readinessprobe with tcp, http, exec + + +Configmaps and Secrets + - Configmaps are k8s object that allow us to seperate the configuration data from + the image content of the pod. + - using this we can inject the environment variables to the pod containers. + - By deafault data is not encrypted in configmaps so it is better to use these for + non-confidential data. + + Create a configmap + 1. Create a file by name "app.properties" + environment=test + database_url="192.168.1.1" + database_password="adjhfgjladhgalhg" + + 2. Load the single config file + kubectl create configmap --from-file configs/app.properties + + Load the multiple config files + kubectl create configmap --from-file configs/ + + Create configmap spec file + apiVersion: v1 + kind: ConfigMap + metadata: + name: test-configmap + data: + environment: test + app: frontend + + 3. To use configmaps to inject env varible in a pod + + apiVersion: v1 + kind: Pod + metadata: + name: nginx-deployment-new + spec: + containers: + - name: nginx + image: nginx:1.14.2 + ports: + - containerPort: 80 + env: + - name: CURRENT_ENV + valueFrom: + configMapKeyRef: + name: test-configmap + key: environment + - name: DB_URL + valueFrom: + configMapKeyRef: + name: test-configmap + key: database_url + + SECRETS + - using secrets we can inject the environment variables to the pod containers in encrypted. + - By deafault secrets data will be in base64 format and we use secrets for confidential data. + + Create a configmap + 1. Create values in base64 format + echo "" | base64 + output: + echo "" | base64 + output: + 2. Load the single config file + kubectl create secret --from-file configs/app.properties + + Load the multiple config files + kubectl create secret --from-file configs/ + Create configmap spec file + apiVersion: v1 + kind: Secret + metadata: + name: test-secret + data: + dburl: + dbpassword: + 3. To use secrets to inject env varible in a pod + apiVersion: v1 + kind: Pod + metadata: + name: nginx-deployment-new + spec: + containers: + - name: nginx + image: nginx:1.14.2 + ports: + - containerPort: 80 + env: + - name: DB_URL + valueFrom: + secretKeyRef: + name: test-secret + key: dburl + - name: DB_PASSWORD + valueFrom: + secretKeyRef: + name: test-secret + key: dbpassword - \ No newline at end of file +How to create a pod on a particular worker pod + 1. Node selector + - Node selector is a way of binding pod to a worker node or nodes based on the + node labels. + - We cannot use any logical expresions type of selection. + + create a label to worker node + kubectl label node = + + Use nodeSelector with the same label created to create pods in the same worker node + apiVersion: apps/v1 + kind: Deployment + metadata: + name: nginx-deployment + spec: + replicas: 6 + selector: + matchLabels: + app: ipapp + template: + metadata: + labels: + app: ipapp + spec: + nodeSelector: + : + containers: + - name: nginx + image: nginx:latest + ports: + - containerPort: 80 + \ No newline at end of file From 336e44cf4826dc9273581c1b7429b3f5474aea7c Mon Sep 17 00:00:00 2001 From: jaintpharsha Date: Sat, 4 Jun 2022 10:02:17 +0530 Subject: [PATCH 33/37] RBAC 04-June-22 --- kubernetes.txt | 127 ++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 126 insertions(+), 1 deletion(-) diff --git a/kubernetes.txt b/kubernetes.txt index e3a4e90..eaeb52e 100644 --- a/kubernetes.txt +++ b/kubernetes.txt @@ -1021,4 +1021,129 @@ How to create a pod on a particular worker pod image: nginx:latest ports: - containerPort: 80 - \ No newline at end of file + +service-controller +endpoint-controller +node-controller +namespace-controller + +Role-Based access control + - accounts + - Roles + - Binding of roles + + Accounts + 1. USER ACCOUNT + it is used for human users to control the access to k8s. + + 2. SERVICE ACCOUNT + - It is used by the applications which need the access to the cluster. + - Any application running inside or outside the cluster need a service account. + + - We use a bearer token to authenticate the Service account. Beares token + is created and attached to SA using secrets. + + To create a service account + kubectl create sa + + Create a token for sa + apiVersion: v1 + kind: Secret + metadata: + name: test-sa-token + annotations: + kubernetes.io/service-account.name: + type: kubernetes.io/service-account-token + + use this account in an application + + apiVersion: v1 + kind: Pod + metadata: + name: nginx-deployment-new + spec: + serviceAccountName: test + containers: + - name: nginx + image: nginx:1.14.2 + ports: + - containerPort: 80 + +Roles + - For a account we can define set of rules to control the access to k8s resource. + - Roles are always userdifined which need to be attached to a account. + - Roles works for only namepsace. Roles are always defined for a namepsace. + + common fields in roles + apiGroups: List of apis to control the access + Subject: User account, serviceaccount or Groups. + Resources: K8S objects on which we want to define this roles + ex: pods, deployments etc... + Verbs: The operations/actions that can be performed. + ex: ["get","list","create","delete","update","watch","patch","proxy"] + + Create a Role + apiVersion: rbac.authorization.k8s.io/v1 + kind: Role + metadata: + namepsace: default + name: test-role + rules: + - apiGroups: [""] + resources: ["pods"] + verbs: ["get", "watch", "list"] + +ClusterRole + - this is cluster wide role + + Create a ClusterRole + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + name: test-cluster-role + rules: + - apiGroups: [""] + resources: ["pods"] + verbs: ["get", "watch", "list"] + + +Rolebinding and ClusterRoleBiniding + - This helps to attach a role to a subject (useraccount, serviceaccount or groups) + - The only differenece is that we use rolebinding to attach role to account and + clusterrolebinding to attach cluster role to account. + + - We use RoleBinding to bind a Role to a ClusterRole + + RoleBinding + apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: + namespace: default + name: read-pods + subjects: + - kind: ServiceAccount + name: test + namespace: default + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: test-role + + ClusterRoleBiniding + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + namespace: default + name: read-pods + subjects: + - kind: ServiceAccount + name: test + namespace: default + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: test-role + + To check the access + kubectl auth can-i list pods --as=system:serviceaccount:: + \ No newline at end of file From 6c955a0b8db4b7b7ff41a0e603c2bd9b11d54922 Mon Sep 17 00:00:00 2001 From: jaintpharsha Date: Tue, 7 Jun 2022 09:40:04 +0530 Subject: [PATCH 34/37] afinity, taints 07-June-22 --- kubernetes.txt | 157 ++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 124 insertions(+), 33 deletions(-) diff --git a/kubernetes.txt b/kubernetes.txt index eaeb52e..cb65024 100644 --- a/kubernetes.txt +++ b/kubernetes.txt @@ -990,37 +990,7 @@ Configmaps and Secrets name: test-secret key: dbpassword -How to create a pod on a particular worker pod - 1. Node selector - - Node selector is a way of binding pod to a worker node or nodes based on the - node labels. - - We cannot use any logical expresions type of selection. - - create a label to worker node - kubectl label node = - - Use nodeSelector with the same label created to create pods in the same worker node - apiVersion: apps/v1 - kind: Deployment - metadata: - name: nginx-deployment - spec: - replicas: 6 - selector: - matchLabels: - app: ipapp - template: - metadata: - labels: - app: ipapp - spec: - nodeSelector: - : - containers: - - name: nginx - image: nginx:latest - ports: - - containerPort: 80 + service-controller endpoint-controller @@ -1106,7 +1076,6 @@ ClusterRole resources: ["pods"] verbs: ["get", "watch", "list"] - Rolebinding and ClusterRoleBiniding - This helps to attach a role to a subject (useraccount, serviceaccount or groups) - The only differenece is that we use rolebinding to attach role to account and @@ -1146,4 +1115,126 @@ Rolebinding and ClusterRoleBiniding To check the access kubectl auth can-i list pods --as=system:serviceaccount:: - \ No newline at end of file + +How to create a pod on a particular worker pod + 1. Node selector + - Node selector is a way of binding pod to a worker node or nodes based on the + node labels. + - We cannot use any logical expresions type of selection. + + create a label to worker node + kubectl label node = + + Use nodeSelector with the same label created to create pods in the same worker node + apiVersion: apps/v1 + kind: Deployment + metadata: + name: nginx-deployment + spec: + replicas: 6 + selector: + matchLabels: + app: ipapp + template: + metadata: + labels: + app: ipapp + spec: + nodeSelector: + : + containers: + - name: nginx + image: nginx:latest + ports: + - containerPort: 80 + + 2. Node afinity and anti-afinity (Inter-pod affinity) + + Node affinity + - nodeselector with logical expresions is affinity. + - using node affinity we can spread pod across worker nodes based on + CPU and RAM capacity(memory-intense mode), Availabilty zone (HA mode). + + - requiredDuringSchedulingIgnoredDuringExecution + The scheduler can't schedule the pod untill unless the rule is met. + + - preferredDuringSchedulingIgnoredDuringExecution + The scheduler tries to fina a node matching the rule, If a matching node + is not available then scheduler still schedules the pod in normal way. + + - IgnoredDuringExecution + If the node labels are changed after the scheduling of pod still the pod + continues to run. + + spec: + containers: + ........ + + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/os + operator: In + values: + - linux + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 1 + preference: + matchExpressions: + - key: label-1 + operator: In + values: + - key-1 + - weight: 50 + preference: + matchExpressions: + - key: label-2 + operator: In + values: + - key-2 + + node Anti-affinity (Inter-pod Affinity) + - This is used to define whether a given pod should or should not be + scheduled on a particular node based on conditional labels. + + spec: + containers: + ........ + + affinity: + nodeAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + IfNotPresent: + - matchExpressions: + - key: + operator: In + values: + - + + 3. Taints and Tolerations + - Taints are used to repel the pods from a specific node. + - we can apply taints to worker nodes which tell scheduler to repel all the pods + except those pods which contains tolerations for the taint. + - 2 operators we can use Equal and Exists (If we use Exists, no value required) + + - We can check the effects also + 1) NoSchedule - This taint means unless a pod with toleration the scheduler + will never schedule the pod. + 2) NoExecute - To delete all the pods except some reuired pods we can use this. + + To taint a worker node + kubectl taint nodes =: + + spec: + tolerations: + - key: env + value: test + effect: NoSchedule + operator: Equal + containers: + - name: nginx + image: nginx:latest + ports: + - containerPort: 80 From 192103f99970df02494be58bb156063c8fc59e77 Mon Sep 17 00:00:00 2001 From: jaintpharsha Date: Tue, 7 Jun 2022 09:52:55 +0530 Subject: [PATCH 35/37] count quota 07-June-22 --- kubernetes.txt | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/kubernetes.txt b/kubernetes.txt index cb65024..65f2ca6 100644 --- a/kubernetes.txt +++ b/kubernetes.txt @@ -1238,3 +1238,36 @@ How to create a pod on a particular worker pod image: nginx:latest ports: - containerPort: 80 + +Resource quotas and limits + How to limit the number of pods to a namepsace ? + how to limit the memory to a pod ? + + Count quota + - This quota is used to limit the max number of obejcts that we can have + in a namepsace. + syntax: + count/. for resources from non-core groups + count/ for resources from the core group + + Below are the list of resources we can have count quota + count/pods + count/persistentvolumeclaims + count/services + count/secrets + count/configmaps + count/replicationcontrollers + count/deployments.apps + count/replicasets.apps + count/statefulsets.apps + count/jobs.batch + count/cronjobs.batch + + ex: + apiVersion: v1 + kind: ResourceQuota + metadata: + name: pod-count-quota + spec: + hard: + scount/pods: "2" \ No newline at end of file From 6a4ad8d5ca4041cd0855c86e8941394911a62758 Mon Sep 17 00:00:00 2001 From: jaintpharsha Date: Wed, 8 Jun 2022 09:56:42 +0530 Subject: [PATCH 36/37] Jobs, Deployment stratergy 08-June-22 --- kubernetes.txt | 239 ++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 238 insertions(+), 1 deletion(-) diff --git a/kubernetes.txt b/kubernetes.txt index 65f2ca6..f2c2ff1 100644 --- a/kubernetes.txt +++ b/kubernetes.txt @@ -1270,4 +1270,241 @@ Resource quotas and limits name: pod-count-quota spec: hard: - scount/pods: "2" \ No newline at end of file + scount/pods: "2" + + Quota on CPU/RAM/Disk + apiVersion: v1 + kind: ResourceQuota + metadata: + name: resource-quota + spec: + hard: + request.cpu: "0.2" + limits.cpu: "0.8" + + 0r + + request.memory: "512Mi" + limits.memory: "800Mi" + + CPU + - 1 cpu, in k8s 1 is equal to 100% to 1 cpu/core and 1 hyperthread. + - if not specified by deafult k8s allocates 0.5 cpu to a pod. + + we can have 0.1, 0.2, ..... 0.9, 1 + + 0.1 cpu = 100m = 1 hundred milli cpu + + Memory + - In k8s resources are measured in bytes. + - The memory should in simple integer value (Fixed point number). + - Representation Ki,Mi,Gi,Ti,Pi,Ei + + apiVersion: v1 + kind: Pod + metadata: + name: nginx-deployment-new + spec: + containers: + - name: nginx + image: nginx:1.14.2 + ports: + - containerPort: 80 + resources: + requests: + memory: "100Mi" + cpu: 0.5 + limits: + memory: "250Mi" + cpu: 1 + +- The main function of a job is to create one or more pods and tracks the success + status of pods. + - Jobs ensure that the specified number of pods is completed successfully and + when the job is completed pods go to the shutdown state and Job goes to completed state. + - Mainly we use jobs to run pod temporarily till the task is completed and to run + tasks parallely. + + apiVersion: batch/v1 + kind: Job + metadata: + name: my-job + spec: + template: + spec: + containers: + - name: busybox + image: busybox + command: ["echo", "This is first job"] + restartPolicy: Never + + restartPolicy + - This is applied to pod not for the Job. + - There are values, + 1. Always + - This is the default restart policy containers will always be restarted if they stop, + even if they completed successfully. + - This policy should be used for applications that always needs to be running. + 2. OnFailure + - will always restart containers only if the container process exits with an error code. + - If the container is determined to be unhealthy by a liveness probe it will be restarted. + - we use this policy for applications that need to run successfully & then stop. + 3. Never + - The pod’s containers will never be restarted, even if the container exits with error code + or a liveness probe fails. + The different type of jobs or common parameters are, + + Completions + - This is the number of times the job to run. default is 1. + - If, completions is 5 then job will run 5 times means 5 pods. + apiVersion: batch/v1 + kind: Job + metadata: + name: my-job + spec: + completions: 5 + template: + spec: + containers: + - name: busybox + image: busybox + command: ["echo", "This is first job"] + restartPolicy: Never + + parallelism + - By default jobs run serialy so to run jobs parallely we need to use the parallelism. + - parallelism is used to set the number of job that need to run prallely. + + apiVersion: batch/v1 + kind: Job + metadata: + name: my-job + spec: + completions: 5 + parallelism: 2 + template: + spec: + containers: + - name: busybox + image: busybox + command: ["echo", "This is first job"] + restartPolicy: Never + + backoffLimit + - If the container is failing for some reason which affects the completion the job, + then still job creates more pods one after another until it succeeds which will + simply put a load on the cluster, in this case backoffLimit is used. + - backoffLimitlimit ensure the number pods to limit after failure. + - backoffLimit: 2, once pods fails for 2 times it won’t create more pods. + + apiVersion: batch/v1 + kind: Job + metadata: + name: my-job + spec: + backoffLimit: 2 + template: + spec: + containers: + - name: busybox + image: busybox + command: ["sleep", "60"] + restartPolicy: Never + + activeDeadlineSecond + - This is used to set the execution time for pod and if pod takes more than this + deadline time then pods will be terminated automatically. + + apiVersion: batch/v1 + kind: Job + metadata: + name: my-job + spec: + activeDeadlineSecond: 20 + template: + spec: + containers: + - name: busybox + image: busybox + command: ["sleep", "60"] + restartPolicy: Never + + Scheduled / CronJob + apiVersion: batch/v1 + kind: CronJob + metadata: + name: hello + spec: + schedule: "* * * * *" + jobTemplate: + spec: + template: + spec: + containers: + - name: hello + image: busybox:1.28 + imagePullPolicy: IfNotPresent + command: + - /bin/sh + - -c + - date; echo Hello from the Kubernetes cluster + restartPolicy: OnFailure + + 1. SuccessfulJobHistoryLimit and FailedJobHistoryLimit + apiVersion: batch/v1 + kind: CronJob + metadata: + name: hello + spec: + schedule: "* * * * *" + successfulJobHistoryLimit: 2 + failedJobHistoryLimit: 1 + jobTemplate: + spec: + template: + spec: + containers: + - name: hello + image: busybox:1.28 + imagePullPolicy: IfNotPresent + command: + - /bin/sh + - -c + - date; echo Hello from the Kubernetes cluster + restartPolicy: OnFailure + +Deployment stratergies. + Rolling Update + - By default deployment in k8s uses rolling update stratergy which means if I want use + this stratergy I don'nt need to specify any parameters in spec file. + - Example: By default k8s automatically decides the percentage of keeing available pods. + usually one out 4 it updates. + + - To overrride the default behaviour + spec: + stratergy: + type: RollingUpdate + rollingUpdate: + maxSurge: 1 + maxUnavailable: 25% + + Recreate + The Recreate stratergy will bring all the old pods down immediately and the creates + new updated pods to match the replica count. + spec: + stratergy: + type: Recreate + + Blue/Green deployment + - We keep 2 sets of similar environment in which one will be live called blue + environment and the one which is not live is called as green. + - we update the new changes to blue environment first which is not live and we + swap/ redirect the traffic from blue to green environment. + - Finally blue environment with new updates will become live and we rename it as + the current new blue environment. + + Canary release + - A canary release is a software testing technique used to reduce the risk of + introducing a new software version into production by gradually rolling out + the change to a small subgroup of users, before rolling it out to the entire + platform/infrastructure. \ No newline at end of file From a72b4f4a81356c747a1a7c6b6cecb4511c50f243 Mon Sep 17 00:00:00 2001 From: jaintpharsha Date: Thu, 9 Jun 2022 10:38:15 +0530 Subject: [PATCH 37/37] multi master, hpa vpa 09-June-22 --- kubernetes.txt | 112 ++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 111 insertions(+), 1 deletion(-) diff --git a/kubernetes.txt b/kubernetes.txt index f2c2ff1..dd11a9a 100644 --- a/kubernetes.txt +++ b/kubernetes.txt @@ -1226,6 +1226,11 @@ How to create a pod on a particular worker pod To taint a worker node kubectl taint nodes =: + + To delete a taint from worker node + (Put a "-" at the end of the command) + kubectl taint nodes =:- + spec: tolerations: @@ -1507,4 +1512,109 @@ Deployment stratergies. - A canary release is a software testing technique used to reduce the risk of introducing a new software version into production by gradually rolling out the change to a small subgroup of users, before rolling it out to the entire - platform/infrastructure. \ No newline at end of file + platform/infrastructure. + +Multi master cluster + What is the size of the k8s cluster ? + - we are manitaining different cluster for different environment + - we have one big cluster and environments are maintained through namepsaces + - always the count of master nodes should a odd number + - we are using loadbalancer / multiple people are working they may dd the worker nodes + so I never kept exact count of nodes but on average we have 20 to 25 worker nodes. + + Why always the number of master nodes is odd number ? + - Based on Quoram calculation (n/2+1) we get same quoram failure rate for odd number + and its next even number of nodes, so it is better to use odd number nodes instead + of even number and we start with minimum 3 master nodes. + - Always tell odd number of nodes (any odd number starting with 3, 5, 7, 9) + - Based on the quorum value we choose only odd number of nodes starting with 3 to + achive better fault tollerance cluster. + +pod eviction + + - Kubernetes evict pods if the node resources are running out such as cpu, RAM and storage. + - Pod with failed state will be evicted first because they may not running but could still + be using cluster resources and then k8s runs decision making based. + + Kubernetes looks at two different reasons to make eviction decision: + 1. QoS (Quality of Service) class. + For every container in the pod: + - There must be a memory limit and a memory request. + - The memory limit must equal the memory request. + - There must be a CPU limit and a CPU request. + - The CPU limit must equal the CPU request. + 2. Priority class. + - A pod's priority class defines the importance of the pod compared to other + pods running in the cluster. + - based on the priority low to high pods will be evicted. + +k8s AutoScale + Horizontal autoscaler + Horizontal Pod Auto-Scaler (HPA) + - HPA is used to automatically scale the number of pods based on deployments, replicasets, + statefulsets or other objects, based on CPU, Memory threshold. + - Automatic scaling of the horizontal pod does not apply to objects that cannot be scaled. + ex: DaemonSets. + - We need metric server as a soruce for autoscalling. + + Metric server + - Metrics Server collects resource metrics from Kubelets and exposes them in Kubernetes + API server through Metrics API for use by Horizontal Pod Autoscaler and Vertical Pod Autoscaler. + - kubectl top command use Metrics API to list the resource utilization of all pods. + - Metrics Server is not meant for non-autoscaling purposes like we wont forward these metrics + data to monitoring tools. + + apiVersion: autoscaling/v2beta2 + kind: HorizontalPodAutoscaler + metadata: + name: php-apache-hps + spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: php-apache + minReplicas: 1 + maxReplicas: 10 + metrics: + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: 50 + + ---------------- or ---------------- + + kubectl autoscale deployment php-apache — cpu-percent=50 — min=1 — max=10 + + To list HPA + kubectl get hpa + + + Vertical Pod Auto-Scaler (VPA) + - vpa automatically adjusts the CPU and Memory attributes for your Pods. + - basically vpa will recreate your pod with the suitable CPU and Memory attributes. + - when we describe vpa, it will show recommendations for the Memory/CPU requests, Limits and it can also automatically + update the limits. + + apiVersion: autoscaling.k8s.io/v1 + kind: VerticalPodAutoscaler + metadata: + name: my-app-vpa + spec: + targetRef: + apiVersion: "apps/v1" + kind: Deployment + name: my-app + updatePolicy: + updateMode: "Auto" + + + Horizontal / Vertical Cluster Auto-Scaler + - Cluster Autoscaler is a tool that automatically adjusts the size of the Kubernetes + cluster when one of the following conditions is true: + 1. some pods failed to run in the cluster due to insufficient resources, + 2. some nodes in the cluster that have been overloaded for an + extended period and their pods can be placed on other existing nodes. + + - Cluster autoscaller tools are mostly provided by public cloud providers. \ No newline at end of file