From 662bf2840cd243271db0292d7cf5feb8142c3bab Mon Sep 17 00:00:00 2001 From: w136712058 <136712058@qq.com> Date: Wed, 30 Jan 2019 14:27:34 +0800 Subject: [PATCH 01/11] support specify user to run job --- README.rst | 3 +++ django_crontab/crontab.py | 52 +++++++++++++++++++++++++++------------ 2 files changed, 39 insertions(+), 16 deletions(-) diff --git a/README.rst b/README.rst index a53b22b..ab41993 100644 --- a/README.rst +++ b/README.rst @@ -108,6 +108,9 @@ CRONJOBS # format 2 ('0 0 1 * *', 'myapp.cron.other_scheduled_job', ['myapp']), + + # format 3 specify user to run job(can avoid authority problem such as file I/O authority) + ('0 0 * * 0', 'myapp.cron.other_scheduled_job [byuser=username]', '>> /tmp/scheduled_job.log'), ('0 0 * * 0', 'django.core.management.call_command', ['dumpdata', 'auth'], {'indent': 4}, '> /home/john/backups/last_sunday_auth_backup.json'), ] diff --git a/django_crontab/crontab.py b/django_crontab/crontab.py index af0db39..dcf44ab 100755 --- a/django_crontab/crontab.py +++ b/django_crontab/crontab.py @@ -7,6 +7,8 @@ import os import tempfile import sys +import traceback +import time from importlib import import_module @@ -16,8 +18,13 @@ string_type = basestring if sys.version_info[0] == 2 else str # flake8: noqa -logger = logging.getLogger(__name__) - +logger=logging.getLogger(__name__) +logger.setLevel(logging.NOTSET) +handler = logging.FileHandler("/tmp/django_crontab.log") +handler.setLevel(logging.NOTSET) +formatter = logging.Formatter('[%(asctime)s:%(filename)s:%(funcName)s:%(lineno)d]%(message)s') +handler.setFormatter(formatter) +logger.addHandler(handler) class Crontab(object): @@ -152,6 +159,8 @@ def run_job(self, job_hash): job_args = job[2] if len(job) > 2 and not isinstance(job[2], string_type) else [] job_kwargs = job[3] if len(job) > 3 else {} + logger.info('Run cron job %s' % str(job)) + lock_file_name = None # if the LOCK_JOBS option is specified in settings if self.settings.LOCK_JOBS: @@ -161,30 +170,41 @@ def run_job(self, job_hash): try: # acquire the lock fcntl.flock(lock_file, fcntl.LOCK_EX | fcntl.LOCK_NB) - except IOError: - logger.warning('Tried to start cron job %s that is already running.', job) + except Exception,e: + logger.error('Tried to start cron job %s that is already running err:%s.' % (str(job),traceback.format_exc())) return - # parse the module and function names from the job - module_path, function_name = job_name.rsplit('.', 1) - # import the module and get the function - module = import_module(module_path) - func = getattr(module, function_name) - # run the function + olduid=os.getuid() + oldgid=os.getgid() try: + job_name=job[1].split('byuser=')[0].strip() + username=job[1].split('byuser=')[1].strip() + if 'byuser' in job[1]: + ids=os.popen("grep -E '^%s' /etc/passwd|awk -F':' '{print $3,$4}'" % username).readlines()[0] + uid=int(ids.split(' ')[0]) + gid=int(ids.split(' ')[1]) + os.setgid(gid) + os.setuid(uid) + + module_path, function_name = job_name.rsplit('.', 1) + module = import_module(module_path) + func = getattr(module, function_name) func(*job_args, **job_kwargs) - except: - logger.exception('Failed to complete cronjob at %s', job) + os.setgid(oldgid) + os.setuid(olduid) + except Exception, e: + os.setgid(oldgid) + os.setuid(olduid) + logger.error('Failed to complete cronjob at %s err:%s' % (str(job)),traceback.format_exc()) # if the LOCK_JOBS option is specified in settings if self.settings.LOCK_JOBS: try: # release the lock fcntl.flock(lock_file, fcntl.LOCK_UN) - except IOError: - logger.exception('Error unlocking %s', lock_file_name) - return - + except Exception,e: + logger.error('Error unlocking %s err:%s' % (lock_file_name,traceback.format_exc())) + return def __hash_job(self, job): """ Builds an md5 hash representing the job From 325f7fc549adc1c231d86ae88c71a0c6c9ce95a8 Mon Sep 17 00:00:00 2001 From: w136712058 <136712058@qq.com> Date: Wed, 30 Jan 2019 16:43:43 +0800 Subject: [PATCH 02/11] support specify user to run job --- django_crontab/crontab.py | 43 ++++++++++++++++----------------------- 1 file changed, 18 insertions(+), 25 deletions(-) diff --git a/django_crontab/crontab.py b/django_crontab/crontab.py index dcf44ab..6c4a713 100755 --- a/django_crontab/crontab.py +++ b/django_crontab/crontab.py @@ -8,7 +8,6 @@ import tempfile import sys import traceback -import time from importlib import import_module @@ -18,13 +17,8 @@ string_type = basestring if sys.version_info[0] == 2 else str # flake8: noqa -logger=logging.getLogger(__name__) -logger.setLevel(logging.NOTSET) -handler = logging.FileHandler("/tmp/django_crontab.log") -handler.setLevel(logging.NOTSET) -formatter = logging.Formatter('[%(asctime)s:%(filename)s:%(funcName)s:%(lineno)d]%(message)s') -handler.setFormatter(formatter) -logger.addHandler(handler) +logger = logging.getLogger(__name__) + class Crontab(object): @@ -159,8 +153,6 @@ def run_job(self, job_hash): job_args = job[2] if len(job) > 2 and not isinstance(job[2], string_type) else [] job_kwargs = job[3] if len(job) > 3 else {} - logger.info('Run cron job %s' % str(job)) - lock_file_name = None # if the LOCK_JOBS option is specified in settings if self.settings.LOCK_JOBS: @@ -170,21 +162,21 @@ def run_job(self, job_hash): try: # acquire the lock fcntl.flock(lock_file, fcntl.LOCK_EX | fcntl.LOCK_NB) - except Exception,e: - logger.error('Tried to start cron job %s that is already running err:%s.' % (str(job),traceback.format_exc())) + except IOError: + logger.warning('Tried to start cron job %s that is already running.', job) return - olduid=os.getuid() - oldgid=os.getgid() + olduid = os.getuid() + oldgid = os.getgid() try: - job_name=job[1].split('byuser=')[0].strip() - username=job[1].split('byuser=')[1].strip() + job_name = job[1].split('byuser=')[0].strip() + username = job[1].split('byuser=')[1].strip() if 'byuser' in job[1]: - ids=os.popen("grep -E '^%s' /etc/passwd|awk -F':' '{print $3,$4}'" % username).readlines()[0] - uid=int(ids.split(' ')[0]) - gid=int(ids.split(' ')[1]) - os.setgid(gid) - os.setuid(uid) + ids = os.popen("grep -E '^%s' /etc/passwd|awk -F':' '{print $3,$4}'" % username).readlines()[0] + uid = int(ids.split(' ')[0]) + gid = int(ids.split(' ')[1]) + os.setgid(gid) + os.setuid(uid) module_path, function_name = job_name.rsplit('.', 1) module = import_module(module_path) @@ -195,16 +187,17 @@ def run_job(self, job_hash): except Exception, e: os.setgid(oldgid) os.setuid(olduid) - logger.error('Failed to complete cronjob at %s err:%s' % (str(job)),traceback.format_exc()) + logger.error('Failed to complete cronjob at %s err:%s' % (str(job)), traceback.format_exc()) # if the LOCK_JOBS option is specified in settings if self.settings.LOCK_JOBS: try: # release the lock fcntl.flock(lock_file, fcntl.LOCK_UN) - except Exception,e: - logger.error('Error unlocking %s err:%s' % (lock_file_name,traceback.format_exc())) - return + except IOError: + logger.exception('Error unlocking %s', lock_file_name) + return + def __hash_job(self, job): """ Builds an md5 hash representing the job From 9c76e07032c7147c2213b78ecdd9cb738574dcec Mon Sep 17 00:00:00 2001 From: w136712058 <136712058@qq.com> Date: Wed, 30 Jan 2019 16:56:41 +0800 Subject: [PATCH 03/11] support specify user to run job --- django_crontab/crontab.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django_crontab/crontab.py b/django_crontab/crontab.py index 6c4a713..d332b84 100755 --- a/django_crontab/crontab.py +++ b/django_crontab/crontab.py @@ -187,7 +187,7 @@ def run_job(self, job_hash): except Exception, e: os.setgid(oldgid) os.setuid(olduid) - logger.error('Failed to complete cronjob at %s err:%s' % (str(job)), traceback.format_exc()) + logger.error('Failed to complete cronjob at %s err:%s' % (str(job), traceback.format_exc()) # if the LOCK_JOBS option is specified in settings if self.settings.LOCK_JOBS: From 81e03123cb92b1e6cc199e1bbd4ff3d3e644f373 Mon Sep 17 00:00:00 2001 From: w136712058 <136712058@qq.com> Date: Wed, 30 Jan 2019 17:14:45 +0800 Subject: [PATCH 04/11] support specify user to run job --- django_crontab/crontab.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/django_crontab/crontab.py b/django_crontab/crontab.py index d332b84..b42bdd5 100755 --- a/django_crontab/crontab.py +++ b/django_crontab/crontab.py @@ -187,7 +187,7 @@ def run_job(self, job_hash): except Exception, e: os.setgid(oldgid) os.setuid(olduid) - logger.error('Failed to complete cronjob at %s err:%s' % (str(job), traceback.format_exc()) + logger.error('Failed to complete cronjob at %s err:%s' % (str(job), traceback.format_exc())) # if the LOCK_JOBS option is specified in settings if self.settings.LOCK_JOBS: @@ -221,4 +221,3 @@ def __get_job_by_hash(self, job_hash): 'No job with hash %s found. It seems the crontab is out of sync with your settings.CRONJOBS. ' 'Run "python manage.py crontab add" again to resolve this issue!' % job_hash ) - From 31ae25d0fec219289dd9ab51a825f9f9706b81c7 Mon Sep 17 00:00:00 2001 From: w136712058 <136712058@qq.com> Date: Wed, 30 Jan 2019 17:34:50 +0800 Subject: [PATCH 05/11] support specify user to run job --- django_crontab/crontab.py | 50 ++++++++++++++++++++++++--------------- 1 file changed, 31 insertions(+), 19 deletions(-) diff --git a/django_crontab/crontab.py b/django_crontab/crontab.py index b42bdd5..8ce90be 100755 --- a/django_crontab/crontab.py +++ b/django_crontab/crontab.py @@ -166,28 +166,40 @@ def run_job(self, job_hash): logger.warning('Tried to start cron job %s that is already running.', job) return - olduid = os.getuid() - oldgid = os.getgid() - try: - job_name = job[1].split('byuser=')[0].strip() - username = job[1].split('byuser=')[1].strip() - if 'byuser' in job[1]: - ids = os.popen("grep -E '^%s' /etc/passwd|awk -F':' '{print $3,$4}'" % username).readlines()[0] - uid = int(ids.split(' ')[0]) - gid = int(ids.split(' ')[1]) - os.setgid(gid) - os.setuid(uid) - + if "byuser=" not in job[1]: + # parse the module and function names from the job module_path, function_name = job_name.rsplit('.', 1) + # import the module and get the function module = import_module(module_path) func = getattr(module, function_name) - func(*job_args, **job_kwargs) - os.setgid(oldgid) - os.setuid(olduid) - except Exception, e: - os.setgid(oldgid) - os.setuid(olduid) - logger.error('Failed to complete cronjob at %s err:%s' % (str(job), traceback.format_exc())) + # run the function + try: + func(*job_args, **job_kwargs) + except: + logger.exception('Failed to complete cronjob at %s', job) + else: + olduid = os.getuid() + oldgid = os.getgid() + try: + job_name = job[1].split('byuser=')[0].strip() + username = job[1].split('byuser=')[1].strip() + if 'byuser' in job[1]: + ids = os.popen("grep -E '^%s' /etc/passwd|awk -F':' '{print $3,$4}'" % username).readlines()[0] + uid = int(ids.split(' ')[0]) + gid = int(ids.split(' ')[1]) + os.setgid(gid) + os.setuid(uid) + + module_path, function_name = job_name.rsplit('.', 1) + module = import_module(module_path) + func = getattr(module, function_name) + func(*job_args, **job_kwargs) + os.setgid(oldgid) + os.setuid(olduid) + except Exception, e: + os.setgid(oldgid) + os.setuid(olduid) + logger.error('Failed to complete cronjob at %s err:%s' % (str(job), traceback.format_exc())) # if the LOCK_JOBS option is specified in settings if self.settings.LOCK_JOBS: From bb650a0d824257719ab9a71830b2e46d25572dab Mon Sep 17 00:00:00 2001 From: w136712058 <136712058@qq.com> Date: Wed, 30 Jan 2019 17:59:03 +0800 Subject: [PATCH 06/11] support specify user to run job --- django_crontab/crontab.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/django_crontab/crontab.py b/django_crontab/crontab.py index 8ce90be..994f56b 100755 --- a/django_crontab/crontab.py +++ b/django_crontab/crontab.py @@ -189,14 +189,13 @@ def run_job(self, job_hash): gid = int(ids.split(' ')[1]) os.setgid(gid) os.setuid(uid) - module_path, function_name = job_name.rsplit('.', 1) module = import_module(module_path) func = getattr(module, function_name) func(*job_args, **job_kwargs) os.setgid(oldgid) os.setuid(olduid) - except Exception, e: + except: os.setgid(oldgid) os.setuid(olduid) logger.error('Failed to complete cronjob at %s err:%s' % (str(job), traceback.format_exc())) From 493b2e6ecd31b72e165a55bf85e8eae6b5e1dc6c Mon Sep 17 00:00:00 2001 From: w136712058 <136712058@qq.com> Date: Wed, 30 Jan 2019 18:19:49 +0800 Subject: [PATCH 07/11] support specify user to run job --- django_crontab/crontab.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/django_crontab/crontab.py b/django_crontab/crontab.py index 994f56b..b2bbd77 100755 --- a/django_crontab/crontab.py +++ b/django_crontab/crontab.py @@ -175,7 +175,7 @@ def run_job(self, job_hash): # run the function try: func(*job_args, **job_kwargs) - except: + except Exception as e: logger.exception('Failed to complete cronjob at %s', job) else: olduid = os.getuid() @@ -195,7 +195,7 @@ def run_job(self, job_hash): func(*job_args, **job_kwargs) os.setgid(oldgid) os.setuid(olduid) - except: + except Exception as e: os.setgid(oldgid) os.setuid(olduid) logger.error('Failed to complete cronjob at %s err:%s' % (str(job), traceback.format_exc())) From 9b7710040efecc5796eb4e58ed4dcb85b56fe83e Mon Sep 17 00:00:00 2001 From: w136712058 <136712058@qq.com> Date: Mon, 23 Dec 2019 12:48:09 +0800 Subject: [PATCH 08/11] fixd:delete unused variable --- django_crontab/crontab.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/django_crontab/crontab.py b/django_crontab/crontab.py index b2bbd77..60df568 100755 --- a/django_crontab/crontab.py +++ b/django_crontab/crontab.py @@ -176,7 +176,7 @@ def run_job(self, job_hash): try: func(*job_args, **job_kwargs) except Exception as e: - logger.exception('Failed to complete cronjob at %s', job) + logger.exception('Failed to complete cronjob at %s err:%s' % (str(job),str(e))) else: olduid = os.getuid() oldgid = os.getgid() @@ -198,7 +198,7 @@ def run_job(self, job_hash): except Exception as e: os.setgid(oldgid) os.setuid(olduid) - logger.error('Failed to complete cronjob at %s err:%s' % (str(job), traceback.format_exc())) + logger.error('Failed to complete cronjob at %s err:%s' % (str(job), str(e))) # if the LOCK_JOBS option is specified in settings if self.settings.LOCK_JOBS: From 812c31fc152bbc3cd6f16b0e3362202783336ffa Mon Sep 17 00:00:00 2001 From: w136712058 <136712058@qq.com> Date: Mon, 23 Dec 2019 13:14:47 +0800 Subject: [PATCH 09/11] delete backstrace import --- django_crontab/crontab.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/django_crontab/crontab.py b/django_crontab/crontab.py index 60df568..7a10e6c 100755 --- a/django_crontab/crontab.py +++ b/django_crontab/crontab.py @@ -7,7 +7,6 @@ import os import tempfile import sys -import traceback from importlib import import_module @@ -176,7 +175,7 @@ def run_job(self, job_hash): try: func(*job_args, **job_kwargs) except Exception as e: - logger.exception('Failed to complete cronjob at %s err:%s' % (str(job),str(e))) + logger.exception('Failed to complete cronjob at %s err:%s' % (str(job), str(e))) else: olduid = os.getuid() oldgid = os.getgid() From 66bfe427f1f410bb7b72ef890237adc91e966e49 Mon Sep 17 00:00:00 2001 From: w136712058 <136712058@qq.com> Date: Mon, 23 Dec 2019 21:40:53 +0800 Subject: [PATCH 10/11] fix django_crontab/crontab.py:17:15: F821 undefined name 'basestring' in 3.x --- django_crontab/crontab.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/django_crontab/crontab.py b/django_crontab/crontab.py index 7a10e6c..357bd90 100755 --- a/django_crontab/crontab.py +++ b/django_crontab/crontab.py @@ -14,7 +14,10 @@ from django_crontab.app_settings import Settings -string_type = basestring if sys.version_info[0] == 2 else str # flake8: noqa +if sys.version_info[0] == 2: + string_type = basestring +else: + string_type = str logger = logging.getLogger(__name__) From 907ca026c65de887ffa986ced2de8638f6835288 Mon Sep 17 00:00:00 2001 From: w136712058 <136712058@qq.com> Date: Fri, 27 Dec 2019 12:18:59 +0800 Subject: [PATCH 11/11] =?UTF-8?q?=E5=88=A0=E9=99=A4=E7=A9=BA=E6=A0=BC?= =?UTF-8?q?=EF=BC=8C=E6=A0=A1=E9=AA=8C=E8=BF=99=E4=B9=88=E4=B8=A5=E6=A0=BC?= =?UTF-8?q?=E5=B9=B2=E5=95=A5=EF=BC=8C=E6=80=8E=E4=B9=88=E9=83=BD=E6=8F=90?= =?UTF-8?q?=E4=BA=A4=E4=B8=8D=E6=88=90=E5=8A=9F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- django_crontab/crontab.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django_crontab/crontab.py b/django_crontab/crontab.py index 357bd90..133ad34 100755 --- a/django_crontab/crontab.py +++ b/django_crontab/crontab.py @@ -17,7 +17,7 @@ if sys.version_info[0] == 2: string_type = basestring else: - string_type = str + string_type = str logger = logging.getLogger(__name__)