腾讯云COS上传、批量删除工具(Python)

2019年9月7日17:12:53来源:Linux运维笔记 评论 353
源码寄售

腾讯云对象存储COS是类似于阿里云OSS,相比OSS,COS提供每月免费额度:存储空间50G、外网访问流量10G(内网免费)、免费读请求100万次、写请求10万次。对网站备份来说不错,但是,腾讯云提供的工具太low,参考阿里云OSS,写了一个cos信息配置、创建目录、上传、批量删除工具(coscmd)

安装Python2.7

此工具在Python2.7上测试通过,建议用OneinStack安装Python2.7,安装路径为:/usr/local/python,命令如下:

cd ~/oneinstack
./addons.sh #选择7,install Let's Encrypt client

配置COS

登陆腾讯云管理后台https://console.qcloud.com/cos/bucket创建bucket,并获取API密钥,在下面配置中一一对应。

cd oneinstack/tools #必须进入该目录执行
wget http://mirrors.linuxeye.com/oneinstack/tools/cos.tgz #v4版本
wget http://mirrors.linuxeye.com/oneinstack/tools/cos_v3.tgz #v3版本,可提交工单升v4
tar xzf cos.tgz
/usr/local/python/bin/python ./coscmd config --appid=[appid] --id=[secret_id] --key=[secret_key] --region=[region] --bucket=[bucket]  #v4配置
/usr/local/python/bin/python ./coscmd config --appid=[appid] --id=[secret_id] --key=[secret_key] --bucket=[bucket]  #v3配置

COSv4版本配置

COSv3版本配置

执行后会将相关信息写到~/.coscredentials,执行其它动作会自动加在改配置。

创建目录

上传文件

批量删除

#!/usr/bin/env python
#coding:utf-8
import sys,os
import datetime
import random
import threading
import time
import datetime
import logging
import ConfigParser
from optparse import OptionParser
from logging.handlers import RotatingFileHandler
from time import strftime, localtime
from time import sleep
from datetime import date
from datetime import timedelta
from cos import CosClient
from cos import UploadFileRequest
from cos import CreateFolderRequest
from cos import DelFileRequest
from cos import DelFolderRequest
from cos import ListFolderRequest
from cos import threadpool
MAX_RETRY_TIMES = 3
LOG_SAVE_EVERY_NUM = 1024
ONE_TASK_DEL_FILE_NUMS = 50
log_level = 1
log_file_name = "del_file.log"
dir_thread_num = 2
file_thread_num = 5
log_out_to_screen = 1
delete_folder_fail_exist = 0
CONFIGFILE = "%s/.coscredentials" % os.path.expanduser('~')
CONFIGSECTION = 'COSCredentials'
HAS_FORK = hasattr(os, 'fork')
HELP = \
'''coscmd:
    config         --appid=[appid] --id=[secret_id] --key=[secret_key] --region=[region] --bucket=[bucket] 
    ls             cosdir
    mkdir          dirname
    put            localfile  cosdir 
    rm(delete,del) object
    '''
CMD_LIST = {}
def cmd_configure(args, options):
    if options.appid is None or options.secret_id is None or options.secret_key is None  or options.region is None or options.bucket is None:
        print("%s miss parameters, use --appid=[appid] --id=[secret_id] --key=[secret_key] --region=[region] --bucket=[bucket] to specify appid/id/key/region/bucket pair" % args[0])
        sys.exit(-1)
    config = ConfigParser.RawConfigParser()
    config.add_section(CONFIGSECTION)
    config.set(CONFIGSECTION, 'appid', options.appid)
    config.set(CONFIGSECTION, 'secret_id', options.secret_id)
    config.set(CONFIGSECTION, 'secret_key', options.secret_key)
    if options.region in ['sh','gz','tj','sgp']:
        config.set(CONFIGSECTION, 'region', options.region)
    else:
        print("input region error, setup use : --region={sh,gz,tj,sgp}")
        sys.exit(-1)
    config.set(CONFIGSECTION, 'bucket', options.bucket)
    cfgfile = open(CONFIGFILE, 'w+')
    config.write(cfgfile)
    print("Your configuration is saved into %s ." % CONFIGFILE)
    cfgfile.close()
    import stat
    os.chmod(CONFIGFILE, stat.S_IREAD | stat.S_IWRITE)
def cmd_loadconfigure():
    config = ConfigParser.ConfigParser()
    config.read(CONFIGFILE)
    global appid
    global secret_id
    global secret_key
    global region
    global bucket
    appid = int(config.get(CONFIGSECTION, 'appid'))
    secret_id = config.get(CONFIGSECTION, 'secret_id').decode('utf-8')
    secret_key = config.get(CONFIGSECTION, 'secret_key').decode('utf-8')
    region = config.get(CONFIGSECTION, 'region')
    bucket = config.get(CONFIGSECTION, 'bucket').decode('utf-8')
    if len(secret_id) == 0 or len(secret_key) == 0 or len(region) == 0 or len(bucket) == 0:
        print("can't get appid/secret_id/secret_key/region/bucket, setup use : config --appid=[appid] --id=[secret_id] --key=[secret_key] --region=[region] --bucket=[bucket]")
        sys.exit(1)
def cmd_lsdir(COSDIR):
    cosdir = COSDIR.decode('utf-8')
    request = ListFolderRequest(bucket, cosdir)
    list_folder_ret = cos_client.list_folder(request)
    if list_folder_ret[u'code'] == 0:
        print(True)
    else:
        print("%s, appid/secret_id/secret_key/region/bucket invalid"% list_folder_ret[u'message'])
def cmd_mkdir(COSDIR):
    cosdir = COSDIR.decode('utf-8')
    request = CreateFolderRequest(bucket, cosdir)
    create_folder_ret = cos_client.create_folder(request)
    if create_folder_ret[u'code'] == 0:
        print("mkdir cos://%s%s OK" % (bucket,COSDIR))
    else:
        print(create_folder_ret[u'message'])
def cmd_put(LOCALFILE,COSFILE):
    localfile = LOCALFILE.decode('utf-8')
    cosfile = COSFILE.decode('utf-8')
    request = UploadFileRequest(bucket, cosfile, localfile)
    request.set_insert_only(0)
    upload_file_ret = cos_client.upload_file(request)
    if upload_file_ret[u'code'] == 0:
        print("put cos://%s%s OK" % (bucket,COSFILE))
    else:
        print(upload_file_ret[u'message'])
def loginit():
    global config
    if (log_file_name == ""):
        return
    log_level = logging.ERROR
    if log_level == 0:
        log_level = logging.DEBUG
    if log_level == 1:
        log_level = logging.INFO
    if log_level == 2:
        log_level = logging.WARNING
        #定义一个RotatingFileHandler,最多备份5个日志文件,每个日志文件最大20M
    logger = logging.getLogger("")
    Rthandler = RotatingFileHandler(log_file_name, maxBytes= 20*1024*1024,backupCount=5)
    Rthandler.setLevel(log_level)
    formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s')
    Rthandler.setFormatter(formatter)
    logger.addHandler(Rthandler)
        #输出日志到屏幕
    console = logging.StreamHandler()
    console.setFormatter(formatter)
    if (log_out_to_screen == 1):
        logger.addHandler(console)
    logger.setLevel(log_level)
    return logger
#日期相关操作
class Dateop():
    @staticmethod
    def isValidDate(str):
        try:
            time.strptime(str, "%Y""%m""%d")
            return True
        except:
            return False
    @staticmethod
    def getdaystr(n=0):
        dt = date.today()-timedelta(days=n)
        tt = dt.timetuple()
        daystr = strftime("%Y""%m""%d",tt)
        return daystr
    @staticmethod
    def cmpDateAgo(t1,t2):
        if (Dateop.isValidDate(t1)==False or Dateop.isValidDate(t2)==False):
            return False
        if (int(t1) <= int (t2)):
            return True
        return False
    @staticmethod
    def isNeedDeleteDir(dirname, n=0):
        if (len(dirname) != 8):
            return False
        if Dateop.isValidDate(dirname) == False:
            return False
        d2 = Dateop.getdaystr(n);
        if Dateop.cmpDateAgo(dirname, d2):
            return True
        return False
#删除文件统计
class FileStat():
    global cos_log
    def __init__(self):
        self.delfilesuccnum = 0
        self.deldirsuccnum = 0
        self.delfilefailnum = 0
        self.deldirfailnum = 0
        self.lock = threading.Lock()
    def addDelFileFailNum(self,num=1):
        self.lock.acquire(1)
        self.delfilefailnum += num
        self.lock.release()
    def addDelDirFailNum(self,num=1):
        self.lock.acquire(1)
        self.deldirfailnum += num
        self.lock.release()
    def addDelDirSuccNum(self, num=1):
        self.lock.acquire(1)
        self.deldirsuccnum += num
        self.lock.release()
    def addDelFileSuccNum(self, num=1):
        self.lock.acquire(1)
        self.delfilesuccnum += num
        self.lock.release()
    def printStat(self):
        msg ="".join(["delfilesuccnum=",str(self.delfilesuccnum),
                ",delfilefailnum=",str(self.delfilefailnum),
                ",deldirsuccnum=",str(self.deldirsuccnum),
                ",deldirfailnum=",str(self.deldirfailnum)])
        print(msg)
    def logStat(self):
        curtime = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
        log = ''.join(["delfilenum=",str(self.delfilesuccnum),
            ",deldirnum=",str(self.deldirsuccnum),",delfilefailnum=",
            str(self.delfilefailnum),",deldirfailnum=",str(self.deldirfailnum)])
        cos_log.info(log)
#执行时间统计
class TimeStat(object):
    global cos_log
    def __init__(self):
        self.start()
    def start(self):
        self.start = datetime.datetime.now()
        self.t1 = time.time()
        msg = "delete task started  ..........."
        cos_log.info(msg)
    def end(self):
        self.end = datetime.datetime.now()
        self.t2 = time.time()
        msg = "delete task ended\n\nrm task finished,\ntimecost:"+str(self.t2-self.t1) + " (s)"
        cos_log.info(msg)
#删除文件列表中的文件
def delfiles(cos_client, bucket, filelist):
    for f in filelist:
        delfile(cos_client, bucket, f)
def delfolders(cos_client, bucket, folderlist):
    for f in folderlist:
        delfolder(cos_client, bucket, f)
#文件夹删除
def delfolder(cos_client, bucket, folder):
    global stat
    global cos_log
    if not folder:
        return 0
    delfolderreq = DelFolderRequest(bucket, folder)
    retry = 0
    while (retry < MAX_RETRY_TIMES):
        ret = cos_client.del_folder(delfolderreq)
        msg = "delfolder fail, bucket="+bucket+",folder="+folder+ret['message']
        if (ret['code'] == 0):
            break
        elif (ret['code'] == -166):
            cos_log.warning(msg)
            break
        #操作太频繁,频控
        elif (ret['code'] == -71):
            sleep(random.randint(1,5))
            cos_log.warning(msg)
            retry += 1
            continue
        #文件夹非空
        elif (ret['code'] == -173):
            break
        else:
            cos_log.warning(msg)
            retry += 1
    if (ret['code'] != 0 and  ret['code'] != -166):
        stat.addDelDirFailNum()
        cos_log.error("delfolder fail, bucket="+bucket+",folder="+folder+ret['message'])
        return ret['code']
    if (ret['code'] == 0):
        stat.addDelDirSuccNum()
        msg = "delfolder success, bucket="+bucket+",folder="+folder
        cos_log.info(msg)
    return 0
#文件删除
def delfile(cos_client, bucket, filepath):
    global stat
    global cos_log
    delfilereq = DelFileRequest(bucket, filepath)
    retry = 0
    while (retry < MAX_RETRY_TIMES):
        ret = cos_client.del_file(delfilereq)
        msg = "delfile fail bucket="+bucket+",file="+filepath+ret['message']
        if (ret['code'] == 0):
            break
        #文件不存在
        elif (ret['code'] == -166):
            cos_log.warning(msg)
            break
        #单目录写操作过快
        elif (ret['code'] == -143):
            sleep(random.randint(1,5))
            cos_log.warning(msg)
            retry += 1
            continue
        #操作太频繁,频控
        elif (ret['code'] == -71):
            sleep(random.randint(1,5))
            cos_log.warning(msg)
            retry += 1
            continue
        else:
            cos_log.warning(msg)
            retry += 1
            continue
    if (ret['code'] != 0 and  ret['code'] != -166):
        stat.addDelFileFailNum()
        cos_log.error("delfile fail, bucket="+bucket+",file="+filepath+ret['message'])
        return ret['code']
    if (ret['code'] == 0):
        stat.addDelFileSuccNum()
        msg = "delfile success, bucket="+bucket+",file="+filepath
        cos_log.info(msg)
    return 0
#递归文件夹进行文件删除
def delete_r(cos_client, bucket, path, thread_pool_file):
    global stat
    global config
    global cos_log
    cos_log.debug("delete_r bucket:"+bucket+",path:"+path)
    context = u""
    #递归文件夹
    while True:
        listfolderreq = ListFolderRequest(bucket, path, 1000, u'', context)
        retry = 0
        while (retry < MAX_RETRY_TIMES):
            listret = cos_client.list_folder(listfolderreq)
            if listret['code'] != 0 :
                retry += 1
                sleep(random.randint(1,3))
                continue
            else:
                break
        if (listret['code'] != 0):
            cos_log.error("delete_r: list folder fail:"+path +",return msg:"+ listret['message'])
            return listret['code']
        if (len(listret['data']['infos']) == 0):
            break;
        filelist = []
        dirlist = []
        for info in listret['data']['infos']:
            fullname = path + info['name']
            #list出来的文件列表中文件夹和文件本身是混杂一起的
            if info.has_key('filesize'):
                filelist.append(fullname)
                if (len(filelist) >= ONE_TASK_DEL_FILE_NUMS):
                    args = [cos_client, bucket, filelist]
                    args_tuple = (args,None)
                    args_list = [args_tuple]
                    requests = threadpool.makeRequests(delfiles, args_list)
                    for req in requests:
                        thread_pool_file.putRequest(req)
                        filelist = []
                        continue
                else:
                    pass
            else:
                dirlist.append(fullname)
                if (len(dirlist) >= ONE_TASK_DEL_FILE_NUMS):
                    args = [cos_client, bucket, dirlist]
                    args_tuple = (args,None)
                    args_list = [args_tuple]
                    requests = threadpool.makeRequests(delfolders, args_list)
                    for req in requests:
                        thread_pool_file.putRequest(req)
                        dirlist = []
                        continue
                else:
                    pass
                pass
        if (len(filelist) > 0):
            args = [cos_client, bucket, filelist]
            args_tuple = (args,None)
            args_list = [args_tuple]
            requests = threadpool.makeRequests(delfiles, args_list)
            for req in requests:
                thread_pool_file.putRequest(req)
                filelist = []
        else:
            pass
        if (len(dirlist) > 0):
            args = [cos_client, bucket, dirlist]
            args_tuple = (args,None)
            args_list = [args_tuple]
            requests = threadpool.makeRequests(delfolders, args_list)
            for req in requests:
                thread_pool_file.putRequest(req)
                filelist = []
        else:
            pass
        cos_log.debug("delete_r thread pool file waiting\n")
        thread_pool_file.wait()
        cos_log.debug("delete_r thread pool file waiting end\n")
        if (listret['data']['listover'] == False):
            context = listret['data']['context']
            continue
        else:
            break
    stat.logStat()
    return 0
#支持Ctrl+C终止程序
class Watcher():
    def __init__(self):
        self.child = os.fork()
        if self.child == 0:
            return
        else:
            self.watch()
    def watch(self):
        global cos_log
        try:
            os.wait()
        except KeyboardInterrupt:
            cos_log.ERROR("ctrl+c terminated rm_recursive.py, exiting...")
            self.kill()
        sys.exit()
    def kill(self):
        try:
            os.kill(self.child, signal.SIGKILL)
        except OSError:
            pass
def cmd_rm(COSDIR):
    global thread_pool
    global cos_log
    global stat
    cos_log = loginit()
    stat = FileStat()
    timestat = TimeStat()
    if HAS_FORK:
      Watcher()
    path = COSDIR.decode('utf-8')
    thread_pool_dir = threadpool.ThreadPool(dir_thread_num)
    thread_pool_file = threadpool.ThreadPool(file_thread_num)
    cos_log.debug("bucket:"+bucket +",path:"+path)
    args = [cos_client, bucket, path, thread_pool_file]
    args_tuple = (args, None)
    args_list = [args_tuple]
    requests = threadpool.makeRequests(delete_r, args_list)
    for req in requests:
        thread_pool_dir.putRequest(req)
    cos_log.debug("thread_pool_dir waiting.....\n")
    thread_pool_dir.wait()
    thread_pool_dir.dismissWorkers(dir_thread_num, True)
    cos_log.debug("thread_pool_dir wait end.....\n")
    timestat.end()
    stat.logStat()
if sys.argv[1] in ['config','ls','mkdir','put','rm','delete','del'] and len(sys.argv) >= 3:
    if sys.argv[1] == 'config':
        parser = OptionParser()
        parser.add_option("-a", "--appid", dest="appid", help="specify appid")
        parser.add_option("-i", "--id", dest="secret_id", help="specify secret id")
        parser.add_option("-k", "--key", dest="secret_key", help="specify secret key")
        parser.add_option("-r", "--region", dest="region", help="specify region")
        parser.add_option("-b", "--bucket", dest="bucket", help="specify bucket")
        (options, args) = parser.parse_args()
        CMD_LIST['config'] = cmd_configure
        CMD_LIST['config'](args, options)
    if sys.argv[1] == 'ls':
        cmd_loadconfigure()
        cos_client = CosClient(appid, secret_id, secret_key, region)
        COSDIR = sys.argv[2]
        cmd_lsdir(COSDIR)
    if sys.argv[1] == 'mkdir':
        cmd_loadconfigure()
        cos_client = CosClient(appid, secret_id, secret_key, region)
        COSDIR = sys.argv[2]
        cmd_mkdir(COSDIR)
    if sys.argv[1] == 'put' and len(sys.argv) == 4:
        cmd_loadconfigure()
        cos_client = CosClient(appid, secret_id, secret_key, region)
        LOCALFILE = sys.argv[2]
        COSFILE = sys.argv[3]
        cmd_put(LOCALFILE,COSFILE)
    if sys.argv[1] in ('rm','delete','del'):
        cmd_loadconfigure()
        cos_client = CosClient(appid, secret_id, secret_key, region)
        COSDIR = sys.argv[2]
        path = COSDIR.decode('utf-8')
        cmd_rm(path)
else:
    print(HELP)
    exit()
  • “爱游博客”微信公众号
  • 关注微信公众号,分享前沿资讯!
  • weinxin
  • “爱游博客”微信小程序
  • 微信扫一扫,资讯唾手可得!
  • weinxin
阿里云首次购买高性能云服务器,享低至2折优惠

发表评论

您必须才能发表评论!