清华大佬耗费三个月吐血整理的几百G的资源，免费分享！....>>>
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 | #!/usr/bin/phthon import os import time import commands import shutil import threading from os.path import join, getsize import MySQLdb as mydb # 备份目录 baseDir = "/data2/backup/backup_data/" # ns 或 wx; 备份后是否要压缩(mydumper 自带压缩功能)，要压缩 True，否则 False. idc = 'ns'; isZip = True # 备份失败是否重试 ，True 则重试,不重试 设 False, retry_sleep 多久后才开始重试(秒) is_errRetryBackup = True; retry_sleep = 300 # 备份日期 backup_date = time.strftime("%Y%m%d") # 备份命令 cmd = "/usr/local/bin/mydumper -h %s -u root -p password? -P %s %s -t 5 -o %s" ''''' 功能描述： 1. mydumper 远程批量备份, 备份列表由配置文件提供 2. 可按要求对备份是否压缩(mydumper 自动压缩) 3. 备份失败允许再尝试备份一次 4. 备份信息写入数据库 ''' def main(): thread_pool = [] # 是否启用压缩 zip = '-c' if isZip == True else '' # 从配置文件读取 ip, 名称, 端口, 并拼凑备份语句 #f = open('/data2/backup/cnf/other_list.cnf', 'r') f = open('/data/other_list.cnf', 'r') for lines in f.readlines(): if (not lines.startswith('#') and len(lines.strip()) > 0): str = lines.split() host, businessName, port, isMaster = str[0], str[1], str[2], str[3] # 业务文件夹不存在则创建 dir = baseDir + '/' + businessName; if (not os.path.exists(dir)): os.makedirs(dir) dir += "/%s%s" % (businessName, backup_date) # 业务目录: dir , 备份目录: dir/name+备份日期 strcmd = cmd % (host, port, zip, dir) th = threading.Thread(target = mydumper, args =(strcmd, dir, businessName, host, port, is_errRetryBackup, int(isMaster))) thread_pool.append(th) if (thread_pool): for t in thread_pool: t.daemon = True t.start() for t in thread_pool: t.join() def mydumper(sCmd, backupDir, businessName, host, port, is_Retry, isMaster): master_host = ""; backup_host = host; name = businessName; port = port; backup_type = 1; file = ""; start_time = ""; stop_time = ""; returncode = 0; file_size = 0; slave_statement = ""; std_err = ""; start_time = time.strftime("%Y%m%d%H%M%S") # 清除可能遗留的备份 if (os.path.exists(backupDir)): shutil.rmtree(backupDir) # 执行备份 returncode, std_err = execute(sCmd) stop_time = time.strftime("%Y%m%d%H%M%S") if (returncode == 0): # 备份 std_err 返回不为空也视为出错。 if (std_err.strip() != ""): returncode = 123456 else: # 获取 change master to 信息，再次校验备份是否有效 returncode, std_err, master_host, slave_statement = statement(backupDir, backup_host, isMaster) if (returncode == 0): file = backupDir if (returncode != 0): # 异常备份标记为: 日期 + _ERR errDir = backupDir + "_ERR" os.rename(backupDir, errDir) file = errDir # 获取备份大小 file_size = getDirsize(file) if (len(std_err) > 255): std_err = std_err[:250] + "..." my_args = [idc, master_host, backup_host, name, port, backup_type, file, start_time, stop_time, returncode, file_size, slave_statement, std_err] # 写入数据库 call_proc(my_args) # 备份失败是否需要重备? 重备允许一次. if (is_Retry == True and returncode != 0): time.sleep(retry_sleep) oldfile = sCmd.split('-o')[1] pos = oldfile.rfind("/") + 1 # 获取备份全路径, 备份文件标记为重备 字样 retry_file = oldfile[:pos] + "ReBackup-" + oldfile[pos:] retryCmd = sCmd.replace(oldfile, retry_file) # 重备开始 mydumper(retryCmd, retry_file.strip(), name, host, port, False, isMaster) def getDirsize(path): # 获取备份文件夹大小 size = 0L for root, dirs, files in os.walk(path): size += sum([getsize(join(root, name)) for name in files]) return (size) def statement(path, backup_host, isMaster): ''''' 功能: 从 metadata 读取change master to 信息 1. 备份过程: 会先生成: metadata.partial, 完成后metadata.partial会重名为: metadata 并写入备份完成时间 2. metadata 分3段: (1) Started dump: 备份开始时间. (2) master 的log-file 和 log-pos 信息 (必有); slave 的host、log-file 和 log-pos 信息 （备机是slave 才有) (3) Finished dump: 备份结束时间 3. 返回错码, master_host 和 change master to 信息 ''' path += "/metadata"; sMetadata = ""; master_host = ""; er_code = 654321; er_info = "%s not exists !!!" % (path) if (os.path.exists(path)): if (isMaster != 1): # 备机是 slave num = 3 sFinds = "SLAVE STATUS" else: num = 2 sFinds = "MASTER STATUS" f = open(path, 'r') rows = f.readlines(); i = 100; lst =[] for s in rows: if (s.find(sFinds) > 0): i = 1; continue if (i <= num): lst.append(s.split(':')[1].strip()) i += 1 if (isMaster == 1): # 备机是 master master_host = backup_host log_file, log_pos = lst; else: # 备机是 slave master_host, log_file, log_pos = lst; er_code = 0 er_info = "" sMetadata = "CHANGE MASTER TO MASTER_HOST='%s',MASTER_LOG_FILE='%s',MASTER_LOG_POS=%s,MASTER_USER='rep_user',MASTER_PASSWORD='meizu.com'" % (master_host, log_file, log_pos ) return (er_code, er_info, master_host, sMetadata) def execute(cmd): ''''' 1.执行 shell 命令 2.返回执行信息 (returncode = 0 则执行成功, std_err 为报错的错误信息) ''' try: returncode, std_err = commands.getstatusoutput(cmd) return (returncode, std_err) except os.error, e: # 异常返回 1001 错误 return (1001, e) def call_proc(my_args): # 备份信息写入数据库 try: conn = mydb.connect(host = '127.0.0.1', user = 'test', passwd = 'zxc/213?', db = 'meizu_item') cur = conn.cursor() cur.callproc('sp_backup_i',[my_args[0], my_args[1], my_args[2], my_args[3], my_args[4], my_args[5], my_args[6], my_args[7], my_args[8], my_args[9], my_args[10], my_args[11], my_args[12]]) conn.commit() except mydb.Error, e: pass # print "Mysql Error %d: %s" % (e.args[0], e.args[1]) finally: cur.close(); conn.close() if __name__ == '__main__': main() |