-
Notifications
You must be signed in to change notification settings - Fork 0
/
butler.py
executable file
·193 lines (160 loc) · 6.3 KB
/
butler.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
#!/usr/bin/env python3
"""
|------------------------------------------------------------
| butler.py
|------------------------------------------------------------
| This script takes the database backups of multiple Laravel
| apps as SQL files. Define the project root and depth in
| .env file and run the script as a cron job.
|
"""
import os
import subprocess
import sys
from pathlib import Path
from datetime import datetime
from dotenv import dotenv_values
from remover import remove_older, run_telescope_pruner
print('Reading parameters from .env file...')
params = dotenv_values(dotenv_path='.env')
def is_compression_enabled():
"""Check if compression is enabled. Default is True."""
try:
return True if int(params['COMPRESS_OUTPUT']) == 1 else False
except KeyError as error:
return True
def remove_older_files():
"""
Check if removing older files is enabled. Default is 0 which depicts false.
:return int The number of days
"""
try:
return int(params['REMOVE_OLDER_FILES'])
except KeyError as error:
return 0
def prune_telescope_data():
"""
Check if telescope data pruning is enabled. Default is -1 which depicts false.
:return int The number of hours
"""
try:
return int(params['PRUNE_TELESCOPE_DATA'])
except KeyError as error:
return -1
def backup_database(project_root, project_depth):
"""
Run mysqldump and backup database as SQL file.
"""
# Validate project directory
try:
ROOT_DIRECTORY = Path(project_root)
if not os.path.exists(ROOT_DIRECTORY):
raise IOError('Error! Project directory not found, aborting...')
except IOError as error:
print(str(error))
sys.exit(0)
# Get output compression setting
compress_output = is_compression_enabled()
# The glob for searching
TARGET_LARAVEL_DIRECTORY = '*' * int(project_depth) + '/*storage/app'
# Search and get the Laravel projects, in this case, we are
# targeting directories which has /storage/app subdirectory,
# i.e. where user uploaded files reside.
print('Searching for directories...\n')
PUBLIC_PATH = ROOT_DIRECTORY.glob(TARGET_LARAVEL_DIRECTORY)
count = 0
for public_directory in PUBLIC_PATH:
# Check if it's not a directory, mainly for safekeeping
if not public_directory.is_dir():
continue
try:
# Load the .env variables as a dictionary
env_file = public_directory.parent.parent / '.env'
env = dotenv_values(dotenv_path=env_file)
# Get necessary keys from .env file
dbname = env['DB_DATABASE']
user = env['DB_USERNAME']
password = env['DB_PASSWORD']
# We will be keeping the backups into a directory named backup inside the app directory
backup_directory = public_directory.joinpath('backup')
os.makedirs(backup_directory, exist_ok=True)
print('\n------------------ [%s] ------------------' % dbname)
# Run telescope data pruner if enabled
if prune_telescope_data() > -1:
run_telescope_pruner(
public_directory.parent.parent, prune_telescope_data())
# Run actual mysqldump process
run(user, password, dbname, backup_directory)
count += 1
print('Success!')
# Remove older backups if feature is enabled
if remove_older_files() > 0 and backup_directory.exists():
remove_older(str(backup_directory),
remove_older_files(), dbname)
print('---------------- //[%s]// ----------------' % dbname)
except Exception as error:
print('Error occurred: ' + str(error))
print('\nBackup generated for %d projects.' % count)
print('\nExecution finished at: ' + str(datetime.now()))
def generate_file_name(dbname, backup_directory):
"""
Generate file name
:param dbname: The database name
:param backup_directory: Location of backup directory
:return: str
"""
now = datetime.now().strftime("%Y-%m-%d_%H_%M_%S")
# Filename consists of the database name and current timestamp as human readable format
filename = dbname + '_' + now + \
('.gz' if is_compression_enabled() else '.sql')
return backup_directory.joinpath(filename)
def run(user, password, dbname, backup_directory):
"""
Run mysqldump process
:param user: Database username
:param password: Database password
:param dbname: Database name
:param backup_directory: Location of backup directory
"""
compress_output = is_compression_enabled()
print("Running mysqldump on {}...".format(dbname))
args = ['mysqldump', '-u{}'.format(user), '-p{}'.format(password), dbname]
with open(generate_file_name(dbname, backup_directory), 'wb') as backup:
process1 = subprocess.Popen(
args, stdout=subprocess.PIPE if compress_output else backup)
if compress_output:
# If compression is enabled, open a second process and compress the output
process2 = subprocess.Popen(
'gzip', stdin=process1.stdout, stdout=backup)
process1.stdout.close()
backup.close()
def start():
"""
Initializes environment variables and runs main program.
"""
try:
valid_depths = (1, 2)
if not params:
raise FileNotFoundError("Error! .env file is not found.")
project_root = params['PROJECT_ROOT']
project_depth = params['PROJECT_DEPTH']
if not project_root:
raise ValueError(
"Error! Project root is not defined. You must set a project root in .env file.")
if int(project_depth) not in valid_depths:
raise ValueError(
"Error! Invalid value defined for depth, value must be between 1 and 2.")
print("""
Project Root: %s
Project Depth: %s
""" % (project_root, project_depth))
backup_database(project_root, project_depth)
return
except ValueError as error:
print(error)
except FileNotFoundError as error:
print(error)
except KeyError as error:
print('\nError! Please define necessary keys in .env file.')
if __name__ == "__main__":
start()