diff --git a/README.md b/README.md index 385b56f..5f76ede 100644 --- a/README.md +++ b/README.md @@ -159,7 +159,7 @@ rm -rf logs/* export PYTHONOPTIMIZE=1 # 解决 celery 不允许创建子进程的问题 -nohup celery -A devops worker -l info -c 3 --max-tasks-per-child 40 --prefetch-multiplier 1 --pidfile logs/celery_worker.pid > logs/celery_worker.log 2>&1 & +nohup celery -A devops worker -l info -c 5 --max-tasks-per-child 200 --prefetch-multiplier 10 --pidfile logs/celery_worker.pid > logs/celery_worker.log 2>&1 & nohup celery -A devops beat -l info --pidfile logs/celery_beat.pid > logs/celery_beat.log 2>&1 & diff --git a/apps/tasks/tasks.py b/apps/tasks/tasks.py index 4098d71..ca73334 100644 --- a/apps/tasks/tasks.py +++ b/apps/tasks/tasks.py @@ -24,6 +24,11 @@ def gen_rand_char(length=16, chars='0123456789zyxwvutsrqponmlkjihgfedcbaZYXWVUTS return ''.join(random.sample(chars, length)) +@app.task() +def show(): + print('ok') + + @app.task() def task_host_update_info(hostinfo): try: diff --git a/deamon.ini b/deamon.ini index 513b22c..1d48a47 100644 --- a/deamon.ini +++ b/deamon.ini @@ -4,7 +4,7 @@ directory=/devops # -c worker进程数,默认 cpu 核数 # --max-tasks-per-child 每个worker执行了多少个任务就会被新的worker替代, # 可以设置小点防止长时间运行Celery有可能发生内存泄露 -command=celery -A devops worker -l info -c 3 --max-tasks-per-child 40 --prefetch-multiplier 1 --pidfile logs/celery_worker.pid +command=celery -A devops worker -l info -c 5 --max-tasks-per-child 200 --prefetch-multiplier 10 --pidfile logs/celery_worker.pid # stdout_logfile=/var/log/celeryd.log # stderr_logfile=/var/log/celeryd.log loglevel=info diff --git a/start_all.sh b/start_all.sh index 6f8c80f..1bb6557 100644 --- a/start_all.sh +++ b/start_all.sh @@ -12,7 +12,7 @@ fi rm -rf logs/*.pid echo 'start celery worker' -nohup celery -A devops worker -l info -c 3 --max-tasks-per-child 40 --prefetch-multiplier 1 --pidfile logs/celery_worker.pid >> logs/celery_worker.log 2>&1 & +nohup celery -A devops worker -l info -c 5 --max-tasks-per-child 200 --prefetch-multiplier 10 --pidfile logs/celery_worker.pid >> logs/celery_worker.log 2>&1 & echo 'start celery beat' nohup celery -A devops beat -l info --pidfile logs/celery_beat.pid >> logs/celery_beat.log 2>&1 & diff --git a/start_celery.py b/start_celery.py index 3a971d1..89d6713 100644 --- a/start_celery.py +++ b/start_celery.py @@ -13,5 +13,5 @@ def __external_cmd(cmd, code="utf8"): if __name__ == '__main__': - __external_cmd('rm -f logs/celery_worker.pid;export PYTHONOPTIMIZE=1; /home/python372/bin/celery -A devops worker -l info -c 3 ' - '--max-tasks-per-child 40 --prefetch-multiplier 1 --pidfile logs/celery_worker.pid') + __external_cmd('rm -f logs/celery_worker.pid;export PYTHONOPTIMIZE=1; /home/python372/bin/celery -A devops worker -l info -c 5 ' + '--max-tasks-per-child 200 --prefetch-multiplier 10 --pidfile logs/celery_worker.pid') diff --git a/start_celery.sh b/start_celery.sh index 71242c5..51543a7 100644 --- a/start_celery.sh +++ b/start_celery.sh @@ -1,4 +1,4 @@ #!/bin/bash export PYTHONOPTIMIZE=1 -/home/python372/bin/celery -A devops worker -l info -c 3 --max-tasks-per-child 40 --prefetch-multiplier 1 +/home/python372/bin/celery -A devops worker -l info -c 5 --max-tasks-per-child 200 --prefetch-multiplier 10 diff --git a/start_real.sh b/start_real.sh index 4ff0702..762ac68 100644 --- a/start_real.sh +++ b/start_real.sh @@ -1,7 +1,7 @@ cd /home/workspace/devops/ && /home/python372/bin/pip3 install -i https://mirrors.aliyun.com/pypi/simple -r requirements.txt -cd /home/workspace/devops/ && export PYTHONOPTIMIZE=1 && /home/python372/bin/celery -A devops worker -l info -c 3 --max-tasks-per-child 40 --prefetch-multiplier 1 --pidfile logs/celery_worker.pid +cd /home/workspace/devops/ && export PYTHONOPTIMIZE=1 && /home/python372/bin/celery -A devops worker -l info -c 5 --max-tasks-per-child 200 --prefetch-multiplier 10 --pidfile logs/celery_worker.pid cd /home/workspace/devops/ && export PYTHONOPTIMIZE=1 && /home/python372/bin/celery -A devops beat -l info --pidfile logs/celery_beat.pid diff --git a/supervisord.conf b/supervisord.conf index 81b344e..eb57bfe 100644 --- a/supervisord.conf +++ b/supervisord.conf @@ -4,7 +4,7 @@ directory=/devops # -c worker进程数,默认 cpu 核数 # --max-tasks-per-child 每个worker执行了多少个任务就会被新的worker替代, # 可以设置小点防止长时间运行Celery有可能发生内存泄露 -command=celery -A devops worker -l info -c 3 --max-tasks-per-child 120 --prefetch-multiplier 1 --pidfile /tmp/%(program_name)s.pid +command=celery -A devops worker -l info -c 5 --max-tasks-per-child 200 --prefetch-multiplier 10 --pidfile /tmp/%(program_name)s.pid stdout_logfile=/tmp/%(program_name)s_stdout.log stderr_logfile=/tmp/%(program_name)s_stderr.log stdout_logfile_maxbytes=52428800 diff --git a/supervisord_all.conf b/supervisord_all.conf index 81b344e..eb57bfe 100644 --- a/supervisord_all.conf +++ b/supervisord_all.conf @@ -4,7 +4,7 @@ directory=/devops # -c worker进程数,默认 cpu 核数 # --max-tasks-per-child 每个worker执行了多少个任务就会被新的worker替代, # 可以设置小点防止长时间运行Celery有可能发生内存泄露 -command=celery -A devops worker -l info -c 3 --max-tasks-per-child 120 --prefetch-multiplier 1 --pidfile /tmp/%(program_name)s.pid +command=celery -A devops worker -l info -c 5 --max-tasks-per-child 200 --prefetch-multiplier 10 --pidfile /tmp/%(program_name)s.pid stdout_logfile=/tmp/%(program_name)s_stdout.log stderr_logfile=/tmp/%(program_name)s_stderr.log stdout_logfile_maxbytes=52428800 diff --git a/supervisord_celery_worker.conf b/supervisord_celery_worker.conf index c9dedfd..1ab3441 100644 --- a/supervisord_celery_worker.conf +++ b/supervisord_celery_worker.conf @@ -4,7 +4,7 @@ directory=/devops # -c worker进程数,默认 cpu 核数 # --max-tasks-per-child 每个worker执行了多少个任务就会被新的worker替代, # 可以设置小点防止长时间运行Celery有可能发生内存泄露 -command=celery -A devops worker -l info -c 3 --max-tasks-per-child 120 --prefetch-multiplier 1 --pidfile /tmp/%(program_name)s.pid +command=celery -A devops worker -l info -c 5 --max-tasks-per-child 200 --prefetch-multiplier 10 --pidfile /tmp/%(program_name)s.pid stdout_logfile=/tmp/%(program_name)s_stdout.log stderr_logfile=/tmp/%(program_name)s_stderr.log stdout_logfile_maxbytes=52428800