通过 python 脚本迁移 Redis 数据

背景

• 需求：需要将的 Redis 数据迁移由云厂商 A 迁移至云厂商 B
• 问题：云版本的 Redis 版本不支持 SYNC、MIGRATE、BGSAVE 等命令，使得许多工具用不了（如 redis-port）

思路

• （1）从 Redis A 获取所有 key，依次导出数据
• （2）将导出的数据加载到 Redis B

实现

示例：将 127.0.0.1:6379 数据迁移至 127.0.0.1:6380

导出脚本 export.py

from optparse import OptionParser
import os
import redis
import logging
import sysmylogger = None
fdata = Nonedef new_logger(logger_name='AppName',level=logging.INFO,to_file=True,log_file_name="app.log",format='[%(asctime)s] %(filename)s:%(lineno)d %(message)s'):if to_file:handler = logging.FileHandler(log_file_name)else:handler = logging.StreamHandler(sys.stdout)handler.setFormatter(logging.Formatter(format))logger = logging.getLogger(logger_name)logger.addHandler(handler)logger.setLevel(level)return loggerdef gen_redis_proto(*args):'''Write out the string in redis protocol so it can be replayed back later'''proto = '*{0}\\r\\n'.format(len(args))for arg in args:proto += '${0}\\r\\n'.format(len(arg))proto += '{0}\\r\\n'.format(arg)return proto# 只取出指定前缀的 key
def match_key(key: str):return key.startswith('normal')def extract(options, fd, logg: logging.Logger):src_r = redis.StrictRedis(host=options.redis_source_url, port=options.redis_source_port, db=options.redis_source_db)all_keys = src_r.keys('*')for key in all_keys:key_type = ''arr = []try:key = key.decode('utf8')if not match_key(key):continuekey_type = src_r.type(key).decode('utf8')if key_type == 'hash':arr.append('HMSET')arr.append(key)for k, v in src_r.hgetall(key).items():arr.append(k.decode('utf8'))arr.append(v.decode('utf8'))elif key_type == 'string':arr.append('SET')arr.append(key)arr.append(src_r.get(key).decode('utf8'))elif key_type == 'set':arr.append('SADD')arr.append(key)arr.extend([v.decode('utf8') for v in src_r.smembers(key)])elif key_type == 'list':arr.append('LPUSH')arr.append(key)arr.extend([v.decode('utf8') for v in src_r.lrange(key, 0, -1)])elif key_type == 'zset':arr.append('ZADD')arr.append(key)for member, score in src_r.zrange(key, 0, -1, withscores=True):arr.append(str(score))arr.append(member.decode('utf8'))else:# TODO 其它的数据类型logg.error('Unsupported key type detected: {}, key: {}'.format(key_type, key))continuefd.write(gen_redis_proto(*arr) + "\n")# 设置 ttlttl = src_r.ttl(key)if ttl != -1:fd.write(gen_redis_proto(*['EXPIRE', key, str(ttl)]) + "\n")except Exception as e:logg.error('Unsupported key type detected: {}, key: {}, error: {}'.format(key_type, key, e.__str__()))if __name__ == '__main__':parser = OptionParser()parser.add_option('-s', '--redis-source-url',action='store',dest='redis_source_url',help='The url of the source redis which is to be cloned [required]')parser.add_option('-p', '--redis-source-port',action='store',dest='redis_source_port',default=6379,type=int,help='The port of the source redis which is to be cloned [required, \default: 6379]')parser.add_option('-n', '--redis-source-db',action='store',dest='redis_source_db',default=0,type=int,help='The db num of the source redis[required, default: 0]')parser.add_option('-o', '--redis-data-output-file-name',action='store',dest='redis_data_output_file_name',default='redis.data',type=str,help='The output file name of the source redis data[required, default: redis.data]')parser.add_option('-l', '--log-file-name',action='store',dest='log_file_name',default='app.log',type=str,help='The log file name[required, default: app.log]')(options, args) = parser.parse_args()if not (options.redis_source_url and options.redis_source_port):parser.error('redis-source-url, redis-source-port are required arguments. Please see help')data_path = options.redis_data_output_file_nameif os.path.exists(data_path):os.remove(data_path)mylogger = new_logger(to_file=True, level=logging.ERROR, log_file_name=options.log_file_name)with open(data_path, 'a+') as fd:extract(options, fd, mylogger)

导出数据

$ python export.py -s 127.0.0.1 -p 6379$ head redis.data
*5\r\n$5\r\nLPUSH\r\n$11\r\nnormalQueue\r\n$3\r\nccc\r\n$2\r\nbb\r\n$3\r\naaa\r\n
*8\r\n$4\r\nSADD\r\n$9\r\nnormalSet\r\n$2\r\ndd\r\n$2\r\nbb\r\n$2\r\ncc\r\n$2\r\nee\r\n$2\r\naa\r\n$2\r\nff\r\n
*3\r\n$6\r\nEXPIRE\r\n$9\r\nnormalSet\r\n$4\r\n1728\r\n
*6\r\n$5\r\nHMSET\r\n$10\r\nnormalHash\r\n$2\r\nk1\r\n$2\r\nv1\r\n$2\r\nk2\r\n$2\r\nv2\r\n
*3\r\n$3\r\nSET\r\n$9\r\nnormalStr\r\n$3\r\nvvv\r\n

导入脚本 load.sh

#!/bin/shwhile read -r line
donohup printf "%b" "$line"| redis-cli -p 6380 --pipe >> load-std.log 2>> load-err.log &
done < $1

导入数据

sh load.sh redis.data

参考

• https://stackoverflow.com/questions/44288974/sync-with-master-failed-err-unknown-command-sync
• https://gist.github.com/jimmyislive/0efd7a6a1c7f7afd73e8#file-clone_redis-py