mod celery worker start parameter

leffss · Aug 8, 2022 · 27c9616 · 27c9616
1 parent e4ec4ab
commit 27c9616
Show file tree

Hide file tree

Showing 10 changed files with 15 additions and 10 deletions.
diff --git a/README.md b/README.md
@@ -159,7 +159,7 @@ rm -rf logs/*
 
 export PYTHONOPTIMIZE=1		# 解决 celery 不允许创建子进程的问题
 
-nohup celery -A devops worker -l info -c 3 --max-tasks-per-child 40 --prefetch-multiplier 1 --pidfile logs/celery_worker.pid > logs/celery_worker.log 2>&1 &
+nohup celery -A devops worker -l info -c 5 --max-tasks-per-child 200 --prefetch-multiplier 10 --pidfile logs/celery_worker.pid > logs/celery_worker.log 2>&1 &
 
 nohup celery -A devops beat -l info --pidfile logs/celery_beat.pid > logs/celery_beat.log 2>&1 &
 

diff --git a/apps/tasks/tasks.py b/apps/tasks/tasks.py
@@ -24,6 +24,11 @@ def gen_rand_char(length=16, chars='0123456789zyxwvutsrqponmlkjihgfedcbaZYXWVUTS
     return ''.join(random.sample(chars, length))
 
 
+@app.task()
+def show():
+    print('ok')
+
+
 @app.task()
 def task_host_update_info(hostinfo):
     try:

diff --git a/deamon.ini b/deamon.ini
@@ -4,7 +4,7 @@ directory=/devops
 # -c worker进程数，默认 cpu 核数
 # --max-tasks-per-child	每个worker执行了多少个任务就会被新的worker替代，
 # 可以设置小点防止长时间运行Celery有可能发生内存泄露
-command=celery -A devops worker -l info -c 3 --max-tasks-per-child 40 --prefetch-multiplier 1 --pidfile logs/celery_worker.pid
+command=celery -A devops worker -l info -c 5 --max-tasks-per-child 200 --prefetch-multiplier 10 --pidfile logs/celery_worker.pid
 # stdout_logfile=/var/log/celeryd.log
 # stderr_logfile=/var/log/celeryd.log
 loglevel=info

diff --git a/start_all.sh b/start_all.sh
@@ -12,7 +12,7 @@ fi
 rm -rf logs/*.pid
 
 echo 'start celery worker'
-nohup celery -A devops worker -l info -c 3 --max-tasks-per-child 40 --prefetch-multiplier 1 --pidfile logs/celery_worker.pid >> logs/celery_worker.log 2>&1 &
+nohup celery -A devops worker -l info -c 5 --max-tasks-per-child 200 --prefetch-multiplier 10 --pidfile logs/celery_worker.pid >> logs/celery_worker.log 2>&1 &
 
 echo 'start celery beat'
 nohup celery -A devops beat -l info --pidfile logs/celery_beat.pid >> logs/celery_beat.log 2>&1 &

diff --git a/start_celery.py b/start_celery.py
@@ -13,5 +13,5 @@ def __external_cmd(cmd, code="utf8"):
 
 
 if __name__ == '__main__':
-    __external_cmd('rm -f logs/celery_worker.pid;export PYTHONOPTIMIZE=1; /home/python372/bin/celery -A devops worker -l info -c 3 '
-                   '--max-tasks-per-child 40 --prefetch-multiplier 1 --pidfile logs/celery_worker.pid')
+    __external_cmd('rm -f logs/celery_worker.pid;export PYTHONOPTIMIZE=1; /home/python372/bin/celery -A devops worker -l info -c 5 '
+                   '--max-tasks-per-child 200 --prefetch-multiplier 10 --pidfile logs/celery_worker.pid')
diff --git a/start_celery.sh b/start_celery.sh
@@ -1,4 +1,4 @@
 #!/bin/bash
 
 export PYTHONOPTIMIZE=1
-/home/python372/bin/celery -A devops worker -l info -c 3 --max-tasks-per-child 40 --prefetch-multiplier 1
+/home/python372/bin/celery -A devops worker -l info -c 5 --max-tasks-per-child 200 --prefetch-multiplier 10
diff --git a/start_real.sh b/start_real.sh
@@ -1,7 +1,7 @@
 
 cd /home/workspace/devops/ && /home/python372/bin/pip3 install -i https://mirrors.aliyun.com/pypi/simple -r requirements.txt
 
-cd /home/workspace/devops/ && export PYTHONOPTIMIZE=1 && /home/python372/bin/celery -A devops worker -l info -c 3 --max-tasks-per-child 40 --prefetch-multiplier 1 --pidfile logs/celery_worker.pid
+cd /home/workspace/devops/ && export PYTHONOPTIMIZE=1 && /home/python372/bin/celery -A devops worker -l info -c 5 --max-tasks-per-child 200 --prefetch-multiplier 10 --pidfile logs/celery_worker.pid
 
 cd /home/workspace/devops/ && export PYTHONOPTIMIZE=1 && /home/python372/bin/celery -A devops beat -l info --pidfile logs/celery_beat.pid
 

diff --git a/supervisord.conf b/supervisord.conf
@@ -4,7 +4,7 @@ directory=/devops
 # -c worker进程数，默认 cpu 核数
 # --max-tasks-per-child	每个worker执行了多少个任务就会被新的worker替代，
 # 可以设置小点防止长时间运行Celery有可能发生内存泄露
-command=celery -A devops worker -l info -c 3 --max-tasks-per-child 120 --prefetch-multiplier 1 --pidfile /tmp/%(program_name)s.pid
+command=celery -A devops worker -l info -c 5 --max-tasks-per-child 200 --prefetch-multiplier 10 --pidfile /tmp/%(program_name)s.pid
 stdout_logfile=/tmp/%(program_name)s_stdout.log
 stderr_logfile=/tmp/%(program_name)s_stderr.log
 stdout_logfile_maxbytes=52428800

diff --git a/supervisord_all.conf b/supervisord_all.conf
@@ -4,7 +4,7 @@ directory=/devops
 # -c worker进程数，默认 cpu 核数
 # --max-tasks-per-child	每个worker执行了多少个任务就会被新的worker替代，
 # 可以设置小点防止长时间运行Celery有可能发生内存泄露
-command=celery -A devops worker -l info -c 3 --max-tasks-per-child 120 --prefetch-multiplier 1 --pidfile /tmp/%(program_name)s.pid
+command=celery -A devops worker -l info -c 5 --max-tasks-per-child 200 --prefetch-multiplier 10 --pidfile /tmp/%(program_name)s.pid
 stdout_logfile=/tmp/%(program_name)s_stdout.log
 stderr_logfile=/tmp/%(program_name)s_stderr.log
 stdout_logfile_maxbytes=52428800

diff --git a/supervisord_celery_worker.conf b/supervisord_celery_worker.conf
@@ -4,7 +4,7 @@ directory=/devops
 # -c worker进程数，默认 cpu 核数
 # --max-tasks-per-child	每个worker执行了多少个任务就会被新的worker替代，
 # 可以设置小点防止长时间运行Celery有可能发生内存泄露
-command=celery -A devops worker -l info -c 3 --max-tasks-per-child 120 --prefetch-multiplier 1 --pidfile /tmp/%(program_name)s.pid
+command=celery -A devops worker -l info -c 5 --max-tasks-per-child 200 --prefetch-multiplier 10 --pidfile /tmp/%(program_name)s.pid
 stdout_logfile=/tmp/%(program_name)s_stdout.log
 stderr_logfile=/tmp/%(program_name)s_stderr.log
 stdout_logfile_maxbytes=52428800