Update cos backup

backup.sh

@@ -38,8 +38,8 @@ DB_COS_BK() {
     ./db_bk.sh $D
     DB_GREP="DB_${D}_`date +%Y`"
     DB_FILE=`ls -lrt $backup_dir | grep ${DB_GREP} | tail -1 | awk '{print $NF}'`
-    ${python_install_dir}/bin/python ./coscmd put $backup_dir/$DB_FILE /`date +%F`/$DB_FILE
-    [ $? -eq 0 ] && ${python_install_dir}/bin/python ./coscmd rm /`date +%F --date="$expired_days days ago"`/ > /dev/null 2>&1
+    ${python_install_dir}/bin/coscmd upload $backup_dir/$DB_FILE /`date +%F`/$DB_FILE
+    [ $? -eq 0 ] && ${python_install_dir}/bin/coscmd delete -r `date +%F --date="$expired_days days ago"` > /dev/null 2>&1
   done
 }
 
@@ -75,8 +75,8 @@ WEB_COS_BK() {
       tar czf $PUSH_FILE ./$W
       popd
     fi
-    ${python_install_dir}/bin/python ./coscmd put $PUSH_FILE /`date +%F`/Web_${W}_$(date +%Y%m%d_%H).tgz
-    [ $? -eq 0 ] && { [ -e "$PUSH_FILE" ] && rm -rf $PUSH_FILE; ${python_install_dir}/bin/python ./coscmd rm /`date +%F --date="$expired_days days ago"`/ > /dev/null 2>&1; }
+    ${python_install_dir}/bin/coscmd upload $PUSH_FILE /`date +%F`/Web_${W}_$(date +%Y%m%d_%H).tgz
+    [ $? -eq 0 ] && { [ -e "$PUSH_FILE" ] && rm -rf $PUSH_FILE; ${python_install_dir}/bin/coscmd delete -r `date +%F --date="$expired_days days ago"` > /dev/null 2>&1; }
   done
 }
 

backup_setup.sh

@@ -168,7 +168,9 @@ fi
 
 if [[ "$DESC_BK" =~ ^[3,5,6]$ ]]; then
   [ ! -e "${python_install_dir}/bin/python" ] && Install_Python
-  [ ! -e "${python_install_dir}/lib/python2.7/site-packages/requests" ] && ${python_install_dir}/bin/pip install requests
+  [ ! -e "${python_install_dir}/lib/coscmd" ] && ${python_install_dir}/bin/pip install coscmd==1.5.4.1 
+  sed -i "/if query_yes_no/{ n; s/^.*/#&/; }" ${python_install_dir}/lib/python2.7/site-packages/coscmd/cos_client.py 
+  sed -i "s/if query_yes_no/#if query_yes_no/" ${python_install_dir}/lib/python2.7/site-packages/coscmd/cos_client.py
   while :; do echo
     echo 'Please select your backup datacenter:'
     echo -e "\t ${CMSG}1${CEND}. 华南(广州)  ${CMSG}2${CEND}. 华北(天津)"
@@ -182,11 +184,11 @@ if [[ "$DESC_BK" =~ ^[3,5,6]$ ]]; then
       echo "${CWARNING}input error! Please only input number 1~5${CEND}"
     fi
   done
-  [ "$Location" == '1' ] && region='gz'
-  [ "$Location" == '2' ] && region='tj'
-  [ "$Location" == '3' ] && region='sh'
-  [ "$Location" == '4' ] && region='cd'
-  [ "$Location" == '5' ] && region='sgp'
+  [ "$Location" == '1' ] && region='cn-south'
+  [ "$Location" == '2' ] && region='cn-north'
+  [ "$Location" == '3' ] && region='cn-east'
+  [ "$Location" == '4' ] && region='cn-southwest'
+  [ "$Location" == '5' ] && region='sg'
   while :; do echo
     read -p "Please enter the Qcloud COS appid: " appid 
     [ -z "$appid" ] && continue
@@ -200,8 +202,9 @@ if [[ "$DESC_BK" =~ ^[3,5,6]$ ]]; then
     read -p "Please enter the Qcloud COS bucket: " bucket 
     [ -z "$bucket" ] && continue
     echo
-    $python_install_dir/bin/python ./tools/coscmd config --appid=$appid --id=$secret_id --key=$secret_key --region=$region --bucket=$bucket >/dev/null 2>&1
-    if [ "`$python_install_dir/bin/python ./tools/coscmd ls /`" == 'True' ];then
+    $python_install_dir/bin/coscmd config -u $appid -a $secret_id -s $secret_key -r $region -b $bucket >/dev/null 2>&1
+    $python_install_dir/bin/coscmd delete oneinstack.test >/dev/null 2>&1
+    if [ $? = 0 ];then
       echo "${CMSG}appid/secret_id/secret_key/region/bucket OK${CEND}"
       echo
       break

tools/cos/__init__.py

@@ -1,10 +0,0 @@
-#!/usr/bin/env python
-# coding=utf-8
-
-from qcloud_cos import CosClient
-from qcloud_cos import UploadFileRequest
-from qcloud_cos import CreateFolderRequest
-from qcloud_cos import DelFileRequest
-from qcloud_cos import DelFolderRequest
-from qcloud_cos import ListFolderRequest
-import threadpool

tools/cos/qcloud_cos/__init__.py

@@ -1,31 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-from .cos_client import CosClient
-from .cos_client import CosConfig
-from .cos_client import CredInfo
-from .cos_request import UploadFileRequest
-from .cos_request import UploadSliceFileRequest
-from .cos_request import UpdateFileRequest
-from .cos_request import UpdateFolderRequest
-from .cos_request import DelFolderRequest
-from .cos_request import DelFileRequest
-from .cos_request import CreateFolderRequest
-from .cos_request import StatFileRequest
-from .cos_request import StatFolderRequest
-from .cos_request import ListFolderRequest
-from .cos_request import DownloadFileRequest
-from .cos_request import MoveFileRequest
-from .cos_auth import Auth
-from .cos_cred import CredInfo
-
-
-import logging
-
-try:
-    from logging import NullHandler
-except ImportError:
-    class NullHandler(logging.Handler):
-        def emit(self, record):
-            pass
-
-logging.getLogger(__name__).addHandler(NullHandler())

tools/cos/qcloud_cos/cos_auth.py

@@ -1,70 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
-import random
-import time
-import urllib
-import hmac
-import hashlib
-import binascii
-import base64
-
-
-class Auth(object):
-    def __init__(self, cred):
-        self.cred = cred
-
-    def app_sign(self, bucket, cos_path, expired, upload_sign=True):
-        appid = self.cred.get_appid()
-        bucket = bucket.encode('utf8')
-        secret_id = self.cred.get_secret_id().encode('utf8')
-        now = int(time.time())
-        rdm = random.randint(0, 999999999)
-        cos_path = urllib.quote(cos_path.encode('utf8'), '~/')
-        if upload_sign:
-            fileid = '/%s/%s%s' % (appid, bucket, cos_path)
-        else:
-            fileid = cos_path
-
-        if expired != 0 and expired < now:
-            expired = now + expired
-
-        sign_tuple = (appid, secret_id, expired, now, rdm, fileid, bucket)
-
-        plain_text = 'a=%s&k=%s&e=%d&t=%d&r=%d&f=%s&b=%s' % sign_tuple
-        secret_key = self.cred.get_secret_key().encode('utf8')
-        sha1_hmac = hmac.new(secret_key, plain_text, hashlib.sha1)
-        hmac_digest = sha1_hmac.hexdigest()
-        hmac_digest = binascii.unhexlify(hmac_digest)
-        sign_hex = hmac_digest + plain_text
-        sign_base64 = base64.b64encode(sign_hex)
-        return sign_base64
-
-    def sign_once(self, bucket, cos_path):
-        """单次签名(针对删除和更新操作)
-
-        :param bucket: bucket名称
-        :param cos_path: 要操作的cos路径, 以'/'开始
-        :return: 签名字符串
-        """
-        return self.app_sign(bucket, cos_path, 0)
-
-    def sign_more(self, bucket, cos_path, expired):
-        """多次签名(针对上传文件，创建目录, 获取文件目录属性, 拉取目录列表)
-
-        :param bucket: bucket名称
-        :param cos_path: 要操作的cos路径, 以'/'开始
-        :param expired: 签名过期时间, UNIX时间戳, 如想让签名在30秒后过期, 即可将expired设成当前时间加上30秒
-        :return: 签名字符串
-        """
-        return self.app_sign(bucket, cos_path, expired)
-
-    def sign_download(self, bucket, cos_path, expired):
-        """下载签名(用于获取后拼接成下载链接，下载私有bucket的文件)
-
-        :param bucket: bucket名称
-        :param cos_path: 要下载的cos文件路径, 以'/'开始
-        :param expired:  签名过期时间, UNIX时间戳, 如想让签名在30秒后过期, 即可将expired设成当前时间加上30秒
-        :return: 签名字符串
-        """
-        return self.app_sign(bucket, cos_path, expired, False)

tools/cos/qcloud_cos/cos_client.py

@@ -1,176 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
-import requests
-from cos_cred import CredInfo
-from cos_config import CosConfig
-from cos_op import FileOp
-from cos_op import FolderOp
-from cos_request import UploadFileRequest
-from cos_request import UploadSliceFileRequest
-from cos_request import UpdateFileRequest
-from cos_request import UpdateFolderRequest
-from cos_request import DelFileRequest
-from cos_request import DelFolderRequest
-from cos_request import CreateFolderRequest
-from cos_request import StatFolderRequest
-from cos_request import StatFileRequest
-from cos_request import ListFolderRequest
-from cos_request import DownloadFileRequest
-try:
-    from requests.packages.urllib3.exceptions import InsecureRequestWarning
-    requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
-except ImportError:
-    pass
-
-
-class CosClient(object):
-    """Cos客户端类"""
-
-    def __init__(self, appid, secret_id, secret_key, region="shanghai"):
-        """ 设置用户的相关信息
-
-        :param appid: appid
-        :param secret_id: secret_id
-        :param secret_key: secret_key
-        """
-        self._cred = CredInfo(appid, secret_id, secret_key)
-        self._config = CosConfig(region=region)
-        self._http_session = requests.session()
-        self._file_op = FileOp(self._cred, self._config, self._http_session)
-        self._folder_op = FolderOp(self._cred, self._config, self._http_session)
-
-    def set_config(self, config):
-        """设置config"""
-        assert isinstance(config, CosConfig)
-        self._config = config
-        self._file_op.set_config(config)
-        self._folder_op.set_config(config)
-
-    def get_config(self):
-        """获取config"""
-        return self._config
-
-    def set_cred(self, cred):
-        """设置用户的身份信息
-
-        :param cred:
-        :return:
-        """
-        assert isinstance(cred, CredInfo)
-        self._cred = cred
-        self._file_op.set_cred(cred)
-        self._folder_op.set_cred(cred)
-
-    def get_cred(self):
-        """获取用户的相关信息
-
-        :return:
-        """
-        return self._cred
-
-    def upload_file(self, request):
-        """ 上传文件(自动根据文件大小，选择上传策略, 强烈推荐使用),上传策略: 8MB以下适用单文件上传, 8MB(含)适用分片上传
-
-        :param request:
-        :return:
-        """
-        assert isinstance(request, UploadFileRequest)
-        return self._file_op.upload_file(request)
-
-    def upload_single_file(self, request):
-        """单文件上传接口, 适用用小文件8MB以下, 最大不得超过20MB, 否则会返回参数错误
-
-        :param request:
-        :return:
-        """
-        assert isinstance(request, UploadFileRequest)
-        return self._file_op.upload_single_file(request)
-
-    def upload_slice_file(self, request):
-        """ 分片上传接口, 适用于大文件8MB及以上
-
-        :param request:
-        :return:
-        """
-        assert isinstance(request, UploadSliceFileRequest)
-        return self._file_op.upload_slice_file(request)
-
-    def del_file(self, request):
-        """ 删除文件
-
-        :param request:
-        :return:
-        """
-        assert isinstance(request, DelFileRequest)
-        return self._file_op.del_file(request)
-
-    def move_file(self, request):
-        return self._file_op.move_file(request)
-
-    def stat_file(self, request):
-        """获取文件属性
-
-        :param request:
-        :return:
-        """
-        assert isinstance(request, StatFileRequest)
-        return self._file_op.stat_file(request)
-
-    def update_file(self, request):
-        """更新文件属性
-
-        :param request:
-        :return:
-        """
-        assert isinstance(request, UpdateFileRequest)
-        return self._file_op.update_file(request)
-
-    def download_file(self, request):
-        assert isinstance(request, DownloadFileRequest)
-        return self._file_op.download_file(request)
-
-    def create_folder(self, request):
-        """创建目录
-
-        :param request:
-        :return:
-        """
-        assert isinstance(request, CreateFolderRequest)
-        return self._folder_op.create_folder(request)
-
-    def del_folder(self, request):
-        """删除目录
-
-        :param request:
-        :return:
-        """
-        assert isinstance(request, DelFolderRequest)
-        return self._folder_op.del_folder(request)
-
-    def stat_folder(self, request):
-        """获取folder属性请求
-
-        :param request:
-        :return:
-        """
-        assert isinstance(request, StatFolderRequest)
-        return self._folder_op.stat_folder(request)
-
-    def update_folder(self, request):
-        """更新目录属性
-
-        :param request:
-        :return:
-        """
-        assert isinstance(request, UpdateFolderRequest)
-        return self._folder_op.update_folder(request)
-
-    def list_folder(self, request):
-        """获取目录下的文件和目录列表
-
-        :param request:
-        :return:
-        """
-        assert isinstance(request, ListFolderRequest)
-        return self._folder_op.list_folder(request)

tools/cos/qcloud_cos/cos_common.py

@@ -1,225 +0,0 @@
-# -*- coding: utf-8 -*-
-
-from __future__ import print_function
-import struct
-import io
-
-try:
-    range = xrange
-except NameError:
-    pass
-
-
-def _left_rotate(n, b):
-    """Left rotate a 32-bit integer n by b bits."""
-    return ((n << b) | (n >> (32 - b))) & 0xffffffff
-
-
-def _process_chunk(chunk, h0, h1, h2, h3, h4):
-    """Process a chunk of data and return the new digest variables."""
-    assert len(chunk) == 64
-
-    w = [0] * 80
-
-    # Break chunk into sixteen 4-byte big-endian words w[i]
-    for i in range(16):
-        w[i] = struct.unpack(b'>I', chunk[i * 4:i * 4 + 4])[0]
-
-    # Extend the sixteen 4-byte words into eighty 4-byte words
-    for i in range(16, 80):
-        w[i] = _left_rotate(w[i - 3] ^ w[i - 8] ^ w[i - 14] ^ w[i - 16], 1)
-
-    # Initialize hash value for this chunk
-    a = h0
-    b = h1
-    c = h2
-    d = h3
-    e = h4
-
-    for i in range(80):
-        if 0 <= i <= 19:
-            # Use alternative 1 for f from FIPS PB 180-1 to avoid bitwise not
-            f = d ^ (b & (c ^ d))
-            k = 0x5A827999
-        elif 20 <= i <= 39:
-            f = b ^ c ^ d
-            k = 0x6ED9EBA1
-        elif 40 <= i <= 59:
-            f = (b & c) | (b & d) | (c & d)
-            k = 0x8F1BBCDC
-        elif 60 <= i <= 79:
-            f = b ^ c ^ d
-            k = 0xCA62C1D6
-
-        a, b, c, d, e = ((_left_rotate(a, 5) + f + e + k + w[i]) & 0xffffffff,
-                         a, _left_rotate(b, 30), c, d)
-
-    # Add this chunk's hash to result so far
-    h0 = (h0 + a) & 0xffffffff
-    h1 = (h1 + b) & 0xffffffff
-    h2 = (h2 + c) & 0xffffffff
-    h3 = (h3 + d) & 0xffffffff
-    h4 = (h4 + e) & 0xffffffff
-
-    return h0, h1, h2, h3, h4
-
-
-class Sha1Hash(object):
-    """A class that mimics that hashlib api and implements the SHA-1 algorithm."""
-
-    name = 'python-sha1'
-    digest_size = 20
-    block_size = 64
-
-    def __init__(self):
-        # Initial digest variables
-        self._h = (
-            0x67452301,
-            0xEFCDAB89,
-            0x98BADCFE,
-            0x10325476,
-            0xC3D2E1F0,
-        )
-
-        # bytes object with 0 <= len < 64 used to store the end of the message
-        # if the message length is not congruent to 64
-        self._unprocessed = b''
-        # Length in bytes of all data that has been processed so far
-        self._message_byte_length = 0
-
-    def update(self, arg):
-        """Update the current digest.
-        This may be called repeatedly, even after calling digest or hexdigest.
-
-        Arguments:
-            arg: bytes, bytearray, or BytesIO object to read from.
-        """
-        if isinstance(arg, (bytes, bytearray)):
-            arg = io.BytesIO(arg)
-
-        # Try to build a chunk out of the unprocessed data, if any
-        chunk = self._unprocessed + arg.read(64 - len(self._unprocessed))
-
-        # Read the rest of the data, 64 bytes at a time
-        while len(chunk) == 64:
-            self._h = _process_chunk(chunk, *self._h)
-            self._message_byte_length += 64
-            chunk = arg.read(64)
-
-        self._unprocessed = chunk
-        return self
-
-    def digest(self):
-        """Produce the final hash value (big-endian) as a bytes object"""
-        return b''.join(struct.pack(b'>I', h) for h in self._produce_digest())
-
-    def hexdigest(self):
-        """Produce the final hash value (big-endian) as a hex string"""
-        return '%08x%08x%08x%08x%08x' % self._produce_digest()
-
-    def inner_digest(self):
-
-        tmp = struct.unpack(">5I", struct.pack("<5I", *self._h))
-        return '%08x%08x%08x%08x%08x' % tmp
-
-    def _produce_digest(self):
-        """Return finalized digest variables for the data processed so far."""
-        # Pre-processing:
-        message = self._unprocessed
-        message_byte_length = self._message_byte_length + len(message)
-
-        # append the bit '1' to the message
-        message += b'\x80'
-
-        # append 0 <= k < 512 bits '0', so that the resulting message length (in bytes)
-        # is congruent to 56 (mod 64)
-        message += b'\x00' * ((56 - (message_byte_length + 1) % 64) % 64)
-
-        # append length of message (before pre-processing), in bits, as 64-bit big-endian integer
-        message_bit_length = message_byte_length * 8
-        message += struct.pack(b'>Q', message_bit_length)
-
-        # Process the final chunk
-        # At this point, the length of the message is either 64 or 128 bytes.
-        h = _process_chunk(message[:64], *self._h)
-        if len(message) == 64:
-            return h
-        return _process_chunk(message[64:], *h)
-
-
-def sha1(data):
-    """SHA-1 Hashing Function
-    A custom SHA-1 hashing function implemented entirely in Python.
-    Arguments:
-        data: A bytes or BytesIO object containing the input message to hash.
-    Returns:
-        A hex SHA-1 digest of the input message.
-    """
-    return Sha1Hash().update(data).hexdigest()
-
-
-class Sha1Util(object):
-
-    @staticmethod
-    def get_sha1_by_slice(file_name, slice_size):
-        """ Get SHA array based on Qcloud Slice Upload Interface
-
-        :param file_name: local file path
-        :param slice_size: slice size in bit
-        :return: sha array like [{“offset”:0, “datalen”:1024,”datasha”:”aaa”}, {}, {}]
-        """
-        from os import path
-
-        with open(file_name, 'rb') as f:
-
-            result = []
-            file_size = path.getsize(file_name)
-            sha1_obj = Sha1Hash()
-            for current_offset in range(0, file_size, slice_size):
-
-                data_length = min(slice_size, file_size - current_offset)
-                sha1_obj.update(f.read(data_length))
-                sha1_val = sha1_obj.inner_digest()
-                result.append({"offset": current_offset, "datalen": data_length, "datasha": sha1_val})
-
-            result[-1]['datasha'] = sha1_obj.hexdigest()
-            return result
-
-
-if __name__ == '__main__':
-    # Imports required for command line parsing. No need for these elsewhere
-    import argparse
-    import sys
-    import os
-
-    # Parse the incoming arguments
-    parser = argparse.ArgumentParser()
-    parser.add_argument('input', nargs='?',
-                        help='input file or message to hash')
-    args = parser.parse_args()
-
-    data = None
-
-    if args.input is None:
-        # No argument given, assume message comes from standard input
-        try:
-            # sys.stdin is opened in text mode, which can change line endings,
-            # leading to incorrect results. Detach fixes this issue, but it's
-            # new in Python 3.1
-            data = sys.stdin.detach()
-        except AttributeError:
-            # Linux ans OSX both use \n line endings, so only windows is a
-            # problem.
-            if sys.platform == "win32":
-                import msvcrt
-
-                msvcrt.setmode(sys.stdin.fileno(), os.O_BINARY)
-            data = sys.stdin
-    elif os.path.isfile(args.input):
-        # An argument is given and it's a valid file. Read it
-        data = open(args.input, 'rb')
-    else:
-        data = args.input
-
-    # Show the final digest
-    print('sha1-digest:', sha1(data))

tools/cos/qcloud_cos/cos_config.py

@@ -1,129 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
-
-class CosRegionInfo(object):
-
-    def __init__(self, region=None, hostname=None, download_hostname=None, *args, **kwargs):
-        self._hostname = None
-        self._download_hostname = None
-
-        if region in ['sh', 'shanghai']:
-            self._hostname = 'sh.file.myqcloud.com'
-            self._download_hostname = 'cossh.myqcloud.com'
-
-        elif region in ['gz', 'guangzhou']:
-            self._hostname = 'gz.file.myqcloud.com'
-            self._download_hostname = 'cosgz.myqcloud.com'
-
-        elif region in ['tj', 'tianjin', 'tianjing']:  # bug: for compact previous release
-            self._hostname = 'tj.file.myqcloud.com'
-            self._download_hostname = 'costj.myqcloud.com'
-
-        elif region in ['sgp', 'singapore']:
-            self._hostname = 'sgp.file.myqcloud.com'
-            self._download_hostname = 'cosspg.myqcloud.com'
-
-        elif region in ['cd', 'chengdu']:
-            self._hostname = 'cd.file.myqcloud.com'
-            self._download_hostname = 'coscd.myqcloud.com'
-
-        elif region is not None:
-            self._hostname = '{region}.file.myqcloud.com'.format(region=region)
-            self._download_hostname = 'cos{region}.myqcloud.com'.format(region=region)
-        else:
-            if hostname and download_hostname:
-                self._hostname = hostname
-                self._download_hostname = download_hostname
-            else:
-                raise ValueError("region or [hostname, download_hostname] must be set, and region should be sh/gz/tj/sgp")
-
-    @property
-    def hostname(self):
-        assert self._hostname is not None
-        return self._hostname
-
-    @property
-    def download_hostname(self):
-        assert self._download_hostname is not None
-        return self._download_hostname
-
-
-class CosConfig(object):
-    """CosConfig 有关cos的配置"""
-
-    def __init__(self, timeout=300, sign_expired=300, enable_https=False, *args, **kwargs):
-        self._region = CosRegionInfo(*args, **kwargs)
-        self._user_agent = 'cos-python-sdk-v4'
-        self._timeout = timeout
-        self._sign_expired = sign_expired
-        self._enable_https = enable_https
-        if self._enable_https:
-            self._protocol = "https"
-        else:
-            self._protocol = "http"
-
-    def get_endpoint(self):
-        """获取域名地址
-
-        :return:
-        """
-        # tmpl = "%s://%s/files/v2"
-        return self._protocol + "://" + self._region.hostname + "/files/v2"
-
-    def get_download_hostname(self):
-        return self._region.download_hostname
-
-    def get_user_agent(self):
-        """获取HTTP头中的user_agent
-
-        :return:
-        """
-        return self._user_agent
-
-    def set_timeout(self, time_out):
-        """设置连接超时, 单位秒
-
-        :param time_out:
-        :return:
-        """
-        assert isinstance(time_out, int)
-        self._timeout = time_out
-
-    def get_timeout(self):
-        """获取连接超时，单位秒
-
-        :return:
-        """
-        return self._timeout
-
-    def set_sign_expired(self, expired):
-        """设置签名过期时间, 单位秒
-
-        :param expired:
-        :return:
-        """
-        assert isinstance(expired, int)
-        self._sign_expired = expired
-
-    def get_sign_expired(self):
-        """获取签名过期时间, 单位秒
-
-        :return:
-        """
-        return self._sign_expired
-
-    @property
-    def enable_https(self):
-        assert self._enable_https is not None
-        return self._enable_https
-
-    @enable_https.setter
-    def enable_https(self, val):
-        if val != self._enable_https:
-            if val:
-                self._enable_https = val
-                self._protocol = "https"
-            else:
-                self._enable_https = val
-                self._protocol = "http"

tools/cos/qcloud_cos/cos_cred.py

@@ -1,36 +0,0 @@
-#!/usr/bin/env python
-# -*- coding:utf-8 -*-
-
-from cos_params_check import ParamCheck
-
-
-class CredInfo(object):
-    """CredInfo用户的身份信息"""
-    def __init__(self, appid, secret_id, secret_key):
-        self._appid = appid
-        self._secret_id = secret_id
-        self._secret_key = secret_key
-        self._param_check = ParamCheck()
-
-    def get_appid(self):
-        return self._appid
-
-    def get_secret_id(self):
-        return self._secret_id
-
-    def get_secret_key(self):
-        return self._secret_key
-
-    def check_params_valid(self):
-        if not self._param_check.check_param_int('appid', self._appid):
-            return False
-        if not self._param_check.check_param_unicode('secret_id', self._secret_id):
-            return False
-        return self._param_check.check_param_unicode('secret_key', self._secret_key)
-
-    def get_err_tips(self):
-        """获取错误信息
-
-        :return:
-        """
-        return self._param_check.get_err_tips()

tools/cos/qcloud_cos/cos_err.py

@@ -1,14 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
-
-class CosErr(object):
-    """sdk错误码"""
-    PARAMS_ERROR = -1  # 参数错误
-    NETWORK_ERROR = -2  # 网络错误
-    SERVER_ERROR = -3  # server端返回错误
-    UNKNOWN_ERROR = -4  # 未知错误
-
-    @staticmethod
-    def get_err_msg(errcode, err_info):
-        return {u'code': errcode, u'message': err_info}

tools/cos/qcloud_cos/cos_op.py

@@ -1,704 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
-import os
-import time
-import json
-import hashlib
-import urllib
-from contextlib import closing
-import cos_auth
-from cos_err import CosErr
-from cos_request import UploadFileRequest
-from cos_request import UploadSliceFileRequest
-from cos_request import UpdateFileRequest
-from cos_request import DelFileRequest
-from cos_request import StatFileRequest
-from cos_request import CreateFolderRequest
-from cos_request import UpdateFolderRequest
-from cos_request import StatFolderRequest
-from cos_request import DelFolderRequest
-from cos_request import ListFolderRequest, DownloadFileRequest, MoveFileRequest
-from cos_common import Sha1Util
-
-from logging import getLogger
-from traceback import format_exc
-
-logger = getLogger(__name__)
-
-
-class BaseOp(object):
-    """
-    BaseOp基本操作类型
-    """
-
-    def __init__(self, cred, config, http_session):
-        """ 初始化类
-
-        :param cred: 用户的身份信息
-        :param config: cos_config配置类
-        :param http_session: http 会话
-        """
-        self._cred = cred
-        self._config = config
-        self._http_session = http_session
-        self._expired_period = self._config.get_sign_expired()
-
-    def set_cred(self, cred):
-        """设置用户的身份信息
-
-        :param cred:
-        :return:
-        """
-        self._cred = cred
-
-    def set_config(self, config):
-        """ 设置config
-
-        :param config:
-        :return:
-        """
-        self._config = config
-        self._expired_period = self._config.get_sign_expired()
-
-    def _build_url(self, bucket, cos_path):
-        """生成url
-
-        :param bucket:
-        :param cos_path:
-        :return:
-        """
-        bucket = bucket.encode('utf8')
-        end_point = self._config.get_endpoint().rstrip('/').encode('utf8')
-        appid = self._cred.get_appid()
-        cos_path = urllib.quote(cos_path.encode('utf8'), '~/')
-        url = '%s/%s/%s%s' % (end_point, appid, bucket, cos_path)
-        return url
-
-    def build_download_url(self, bucket, cos_path, sign):
-        # Only support http now
-        appid = self._cred.get_appid()
-        hostname = self._config.get_download_hostname()
-        cos_path = urllib.quote(cos_path)
-        url_tmpl = 'http://{bucket}-{appid}.{hostname}{cos_path}?sign={sign}'
-
-        return url_tmpl.format(bucket=bucket, appid=appid, hostname=hostname, cos_path=cos_path, sign=sign)
-
-    def send_request(self, method, bucket, cos_path, **kwargs):
-        """ 发送http请求
-
-        :param method:
-        :param bucket:
-        :param cos_path:
-        :param args:
-        :return:
-        """
-        url = self._build_url(bucket, cos_path)
-        logger.debug("sending request, method: %s, bucket: %s, cos_path: %s" % (method, bucket, cos_path))
-
-        try:
-            if method == 'POST':
-                http_resp = self._http_session.post(url, verify=False, **kwargs)
-            else:
-                http_resp = self._http_session.get(url, verify=False, **kwargs)
-
-            status_code = http_resp.status_code
-            if status_code < 500:
-                return http_resp.json()
-            else:
-                logger.warning("request failed, response message: %s" % http_resp.text)
-                err_detail = 'url:%s, status_code:%d' % (url, status_code)
-                return CosErr.get_err_msg(CosErr.NETWORK_ERROR, err_detail)
-        except Exception as e:
-            logger.exception("request failed, return SERVER_ERROR")
-            err_detail = 'url:%s, exception:%s traceback:%s' % (url, str(e), format_exc())
-            return CosErr.get_err_msg(CosErr.SERVER_ERROR, err_detail)
-
-    def _check_params(self, request):
-        """检查用户输入参数, 检查通过返回None, 否则返回一个代表错误原因的dict
-
-        :param request:
-        :return:
-        """
-        if not self._cred.check_params_valid():
-            return CosErr.get_err_msg(CosErr.PARAMS_ERROR, self._cred.get_err_tips())
-        if not request.check_params_valid():
-            return CosErr.get_err_msg(CosErr.PARAMS_ERROR, request.get_err_tips())
-        return None
-
-    def del_base(self, request):
-        """删除文件或者目录, is_file_op为True表示是文件操作
-
-        :param request:
-        :return:
-        """
-        check_params_ret = self._check_params(request)
-        if check_params_ret is not None:
-            return check_params_ret
-
-        auth = cos_auth.Auth(self._cred)
-        bucket = request.get_bucket_name()
-        cos_path = request.get_cos_path()
-        sign = auth.sign_once(bucket, cos_path)
-
-        http_header = dict()
-        http_header['Authorization'] = sign
-        http_header['Content-Type'] = 'application/json'
-        http_header['User-Agent'] = self._config.get_user_agent()
-
-        http_body = {'op': 'delete'}
-
-        timeout = self._config.get_timeout()
-        return self.send_request('POST', bucket, cos_path, headers=http_header, data=json.dumps(http_body), timeout=timeout)
-
-    def stat_base(self, request):
-        """获取文件和目录的属性
-
-        :param request:
-        :return:
-        """
-        check_params_ret = self._check_params(request)
-        if check_params_ret is not None:
-            return check_params_ret
-
-        auth = cos_auth.Auth(self._cred)
-        bucket = request.get_bucket_name()
-        cos_path = request.get_cos_path()
-        expired = int(time.time()) + self._expired_period
-        sign = auth.sign_more(bucket, cos_path, expired)
-
-        http_header = dict()
-        http_header['Authorization'] = sign
-        http_header['User-Agent'] = self._config.get_user_agent()
-
-        http_body = dict()
-        http_body['op'] = 'stat'
-
-        timeout = self._config.get_timeout()
-        return self.send_request('GET', bucket, cos_path, headers=http_header, params=http_body, timeout=timeout)
-
-
-class FileOp(BaseOp):
-    """FileOp 文件相关操作"""
-
-    def __init__(self, cred, config, http_session):
-        """ 初始化类
-
-        :param cred: 用户的身份信息
-        :param config: cos_config配置类
-        :param http_session: http 会话
-        """
-        BaseOp.__init__(self, cred, config, http_session)
-        # 单文件上传的最大上限是20MB
-        self.max_single_file = 20 * 1024 * 1024
-
-    @staticmethod
-    def _sha1_content(content):
-        """获取content的sha1
-
-        :param content:
-        :return:
-        """
-        sha1_obj = hashlib.sha1()
-        sha1_obj.update(content)
-        return sha1_obj.hexdigest()
-
-    def update_file(self, request):
-        """更新文件
-
-        :param request:
-        :return:
-        """
-        assert isinstance(request, UpdateFileRequest)
-        logger.debug("request: " + str(request.get_custom_headers()))
-        check_params_ret = self._check_params(request)
-        if check_params_ret is not None:
-            return check_params_ret
-
-        logger.debug("params verify successfully")
-        auth = cos_auth.Auth(self._cred)
-        bucket = request.get_bucket_name()
-        cos_path = request.get_cos_path()
-        sign = auth.sign_once(bucket, cos_path)
-
-        http_header = dict()
-        http_header['Authorization'] = sign
-        http_header['Content-Type'] = 'application/json'
-        http_header['User-Agent'] = self._config.get_user_agent()
-
-        http_body = dict()
-        http_body['op'] = 'update'
-
-        if request.get_biz_attr() is not None:
-            http_body['biz_attr'] = request.get_biz_attr()
-
-        if request.get_authority() is not None:
-            http_body['authority'] = request.get_authority()
-
-        if request.get_custom_headers() is not None and len(request.get_custom_headers()) is not 0:
-            http_body['custom_headers'] = request.get_custom_headers()
-        logger.debug("Update Request Header: " + json.dumps(http_body))
-        timeout = self._config.get_timeout()
-        return self.send_request('POST', bucket, cos_path, headers=http_header, data=json.dumps(http_body), timeout=timeout)
-
-    def del_file(self, request):
-        """删除文件
-
-        :param request:
-        :return:
-        """
-        assert isinstance(request, DelFileRequest)
-        return self.del_base(request)
-
-    def stat_file(self, request):
-        """获取文件的属性
-
-        :param request:
-        :return:
-        """
-        assert isinstance(request, StatFileRequest)
-        return self.stat_base(request)
-
-    def upload_file(self, request):
-        """上传文件, 根据用户的文件大小,选择单文件上传和分片上传策略
-
-        :param request:
-        :return:
-        """
-        assert isinstance(request, UploadFileRequest)
-        check_params_ret = self._check_params(request)
-        if check_params_ret is not None:
-            return check_params_ret
-
-        local_path = request.get_local_path()
-        file_size = os.path.getsize(local_path)
-
-        suit_single_file_zie = 8 * 1024 * 1024
-        if file_size < suit_single_file_zie:
-            return self.upload_single_file(request)
-        else:
-            bucket = request.get_bucket_name()
-            cos_path = request.get_cos_path()
-            local_path = request.get_local_path()
-            slice_size = 1024 * 1024
-            biz_attr = request.get_biz_attr()
-            upload_slice_request = UploadSliceFileRequest(bucket, cos_path, local_path, slice_size, biz_attr)
-            upload_slice_request.set_insert_only(request.get_insert_only())
-            return self.upload_slice_file(upload_slice_request)
-
-    def upload_single_file(self, request):
-        """ 单文件上传
-
-        :param request:
-        :return:
-        """
-        assert isinstance(request, UploadFileRequest)
-        check_params_ret = self._check_params(request)
-        if check_params_ret is not None:
-            return check_params_ret
-
-        local_path = request.get_local_path()
-        file_size = os.path.getsize(local_path)
-        # 判断文件是否超过单文件最大上限, 如果超过则返回错误
-        # 并提示用户使用别的接口
-        if file_size > self.max_single_file:
-            return CosErr.get_err_msg(CosErr.NETWORK_ERROR, 'file is too big, please use upload_file interface')
-
-        auth = cos_auth.Auth(self._cred)
-        bucket = request.get_bucket_name()
-        cos_path = request.get_cos_path()
-        expired = int(time.time()) + self._expired_period
-        sign = auth.sign_more(bucket, cos_path, expired)
-
-        http_header = dict()
-        http_header['Authorization'] = sign
-        http_header['User-Agent'] = self._config.get_user_agent()
-
-        with open(local_path, 'rb') as f:
-            file_content = f.read()
-
-        http_body = dict()
-        http_body['op'] = 'upload'
-        http_body['filecontent'] = file_content
-        http_body['sha'] = FileOp._sha1_content(file_content)
-        http_body['biz_attr'] = request.get_biz_attr()
-        http_body['insertOnly'] = str(request.get_insert_only())
-
-        timeout = self._config.get_timeout()
-        ret = self.send_request('POST', bucket, cos_path, headers=http_header, files=http_body, timeout=timeout)
-
-        if request.get_insert_only() != 0:
-            return ret
-
-        if ret[u'code'] == 0:
-            return ret
-
-        # try to delete object, and re-post request
-        del_request = DelFileRequest(bucket_name=request.get_bucket_name(), cos_path=request.get_cos_path())
-        ret = self.del_file(del_request)
-        if ret[u'code'] == 0:
-            return self.send_request('POST', bucket, cos_path, headers=http_header, files=http_body, timeout=timeout)
-        else:
-            return ret
-
-    def _upload_slice_file(self, request):
-        assert isinstance(request, UploadSliceFileRequest)
-        check_params_ret = self._check_params(request)
-        if check_params_ret is not None:
-            return check_params_ret
-
-        local_path = request.get_local_path()
-        slice_size = request.get_slice_size()
-        enable_sha1 = request.enable_sha1
-
-        if enable_sha1 is True:
-            sha1_by_slice_list = Sha1Util.get_sha1_by_slice(local_path, slice_size)
-            request.sha1_list = sha1_by_slice_list
-            request.sha1_content = sha1_by_slice_list[-1]["datasha"]
-        else:
-            request.sha1_list = None
-            request.sha1_content = None
-
-        control_ret = self._upload_slice_control(request)
-
-        # 表示控制分片已经产生错误信息
-        if control_ret[u'code'] != 0:
-            return control_ret
-
-        # 命中秒传
-        if u'access_url' in control_ret[u'data']:
-            return control_ret
-
-        local_path = request.get_local_path()
-        file_size = os.path.getsize(local_path)
-        if u'slice_size' in control_ret[u'data']:
-            slice_size = control_ret[u'data'][u'slice_size']
-        offset = 0
-        session = control_ret[u'data'][u'session']
-        # ?concurrency
-        if request._max_con <= 1 or (
-                u'serial_upload' in control_ret[u'data'] and control_ret[u'data'][u'serial_upload'] == 1):
-
-            logger.info("upload file serially")
-            slice_idx = 0
-            with open(local_path, 'rb') as local_file:
-
-                while offset < file_size:
-                    file_content = local_file.read(slice_size)
-
-                    data_ret = self._upload_slice_data(request, file_content, session, offset)
-
-                    if data_ret[u'code'] == 0:
-                        if u'access_url' in data_ret[u'data']:
-                            return data_ret
-                    else:
-                        return data_ret
-
-                    offset += slice_size
-                    slice_idx += 1
-        else:
-            logger.info('upload file concurrently')
-            from threadpool import SimpleThreadPool
-            pool = SimpleThreadPool(request._max_con)
-
-            slice_idx = 0
-            with open(local_path, 'rb') as local_file:
-
-                while offset < file_size:
-                    file_content = local_file.read(slice_size)
-
-                    pool.add_task(self._upload_slice_data, request, file_content, session, offset)
-
-                    offset += slice_size
-                    slice_idx += 1
-
-            pool.wait_completion()
-            result = pool.get_result()
-            if not result['success_all']:
-                return {u'code': 1, u'message': str(result)}
-
-        data_ret = self._upload_slice_finish(request, session, file_size)
-        return data_ret
-
-    def upload_slice_file(self, request):
-        """分片文件上传(串行)
-
-        :param request:
-        :return:
-        """
-        ret = self._upload_slice_file(request)
-
-        if ret[u'code'] == 0:
-            return ret
-
-        if request.get_insert_only() == 0:
-            del_request = DelFileRequest(request.get_bucket_name(), request.get_cos_path())
-            ret = self.del_file(del_request)
-            if ret[u'code'] == 0:
-                return self._upload_slice_file(request)
-            else:
-                return ret
-        else:
-            return ret
-
-    def _upload_slice_finish(self, request, session, filesize):
-        auth = cos_auth.Auth(self._cred)
-        bucket = request.get_bucket_name()
-        cos_path = request.get_cos_path()
-        expired = int(time.time()) + self._expired_period
-        sign = auth.sign_more(bucket, cos_path, expired)
-
-        http_header = dict()
-        http_header['Authorization'] = sign
-        http_header['User-Agent'] = self._config.get_user_agent()
-
-        http_body = dict()
-        http_body['op'] = "upload_slice_finish"
-        http_body['session'] = session
-        http_body['filesize'] = str(filesize)
-        if request.sha1_list is not None:
-            http_body['sha'] = request.sha1_list[-1]["datasha"]
-        timeout = self._config.get_timeout()
-
-        return self.send_request('POST', bucket, cos_path, headers=http_header, files=http_body, timeout=timeout)
-
-    def _upload_slice_control(self, request):
-        """串行分片第一步, 上传控制分片
-
-        :param request:
-        :return:
-        """
-        auth = cos_auth.Auth(self._cred)
-        bucket = request.get_bucket_name()
-        cos_path = request.get_cos_path()
-        expired = int(time.time()) + self._expired_period
-        sign = auth.sign_more(bucket, cos_path, expired)
-
-        http_header = dict()
-        http_header['Authorization'] = sign
-        http_header['User-Agent'] = self._config.get_user_agent()
-
-        local_path = request.get_local_path()
-        file_size = os.path.getsize(local_path)
-        slice_size = request.get_slice_size()
-        biz_atrr = request.get_biz_attr()
-
-        http_body = dict()
-        http_body['op'] = 'upload_slice_init'
-        if request.enable_sha1:
-            http_body['sha'] = request.sha1_list[-1]["datasha"]
-            http_body['uploadparts'] = json.dumps(request.sha1_list)
-        http_body['filesize'] = str(file_size)
-        http_body['slice_size'] = str(slice_size)
-        http_body['biz_attr'] = biz_atrr
-        http_body['insertOnly'] = str(request.get_insert_only())
-
-        timeout = self._config.get_timeout()
-        return self.send_request('POST', bucket, cos_path, headers=http_header, files=http_body, timeout=timeout)
-
-    def _upload_slice_data(self, request, file_content, session, offset, retry=3):
-        """串行分片第二步, 上传数据分片
-
-        :param request:
-        :param file_content:
-        :param session:
-        :param offset:
-        :return:
-        """
-        bucket = request.get_bucket_name()
-        cos_path = request.get_cos_path()
-        auth = cos_auth.Auth(self._cred)
-        expired = int(time.time()) + self._expired_period
-        sign = auth.sign_more(bucket, cos_path, expired)
-
-        http_header = dict()
-        http_header['Authorization'] = sign
-        http_header['User-Agent'] = self._config.get_user_agent()
-
-        http_body = dict()
-        http_body['op'] = 'upload_slice_data'
-        http_body['filecontent'] = file_content
-        http_body['session'] = session
-        http_body['offset'] = str(offset)
-        if request.sha1_content is not None:
-            http_body['sha'] = request.sha1_content
-
-        timeout = self._config.get_timeout()
-
-        for _ in range(retry):
-            ret = self.send_request('POST', bucket, cos_path, headers=http_header, files=http_body, timeout=timeout)
-            if ret['code'] == 0:
-                return ret
-        else:
-            return ret
-
-    def __download_url(self, uri, filename, headers):
-        session = self._http_session
-
-        with closing(session.get(uri, stream=True, timeout=30, headers=headers)) as ret:
-            if ret.status_code in [200, 206]:
-
-                if 'Content-Length' in ret.headers:
-                    content_len = int(ret.headers['Content-Length'])
-                else:
-                    raise IOError("download failed without Content-Length header")
-
-                file_len = 0
-                with open(filename, 'wb') as f:
-                    for chunk in ret.iter_content(chunk_size=1024):
-                        if chunk:
-                            file_len += len(chunk)
-                            f.write(chunk)
-                    f.flush()
-                if file_len != content_len:
-                    raise IOError("download failed with incomplete file")
-            else:
-                raise IOError("download failed with status code:" + str(ret.status_code))
-
-    def download_file(self, request):
-        assert isinstance(request, DownloadFileRequest)
-
-        auth = cos_auth.Auth(self._cred)
-        sign = auth.sign_download(request.get_bucket_name(), request.get_cos_path(), self._config.get_sign_expired())
-        url = self.build_download_url(request.get_bucket_name(), request.get_cos_path(), sign)
-        logger.info("Uri is %s" % url)
-        try:
-            self.__download_url(url, request._local_filename, request._custom_headers)
-            return {u'code': 0, u'message': "download successfully"}
-        except Exception as e:
-            return {u'code': 1, u'message': "download failed, exception: " + str(e)}
-
-    def __move_file(self, request):
-
-        auth = cos_auth.Auth(self._cred)
-        bucket = request.get_bucket_name()
-        cos_path = request.get_cos_path()
-        sign = auth.sign_once(bucket, cos_path)
-
-        http_header = dict()
-        http_header['Authorization'] = sign
-        http_header['User-Agent'] = self._config.get_user_agent()
-
-        http_body = dict()
-        http_body['op'] = 'move'
-        http_body['dest_fileid'] = request.dest_path
-        http_body['to_over_write'] = str(1 if request.overwrite else 0)
-
-        timeout = self._config.get_timeout()
-        return self.send_request('POST', bucket, cos_path, headers=http_header, params=http_body, timeout=timeout)
-
-    def move_file(self, request):
-
-        assert isinstance(request, MoveFileRequest)
-        return self.__move_file(request)
-
-
-class FolderOp(BaseOp):
-    """FolderOp 目录相关操作"""
-    def __init__(self, cred, config, http_session):
-        BaseOp.__init__(self, cred, config, http_session)
-
-    def update_folder(self, request):
-        """更新目录
-
-        :param request:
-        :return:
-        """
-        assert isinstance(request, UpdateFolderRequest)
-        check_params_ret = self._check_params(request)
-        if check_params_ret is not None:
-            return check_params_ret
-
-        auth = cos_auth.Auth(self._cred)
-        bucket = request.get_bucket_name()
-        cos_path = request.get_cos_path()
-        sign = auth.sign_once(bucket, cos_path)
-
-        http_header = dict()
-        http_header['Authorization'] = sign
-        http_header['Content-Type'] = 'application/json'
-        http_header['User-Agent'] = self._config.get_user_agent()
-
-        http_body = dict()
-        http_body['op'] = 'update'
-        http_body['biz_attr'] = request.get_biz_attr()
-
-        timeout = self._config.get_timeout()
-        return self.send_request('POST', bucket, cos_path, headers=http_header, data=json.dumps(http_body), timeout=timeout)
-
-    def del_folder(self, request):
-        """删除目录
-
-        :param request:
-        :return:
-        """
-        assert isinstance(request, DelFolderRequest)
-        return self.del_base(request)
-
-    def stat_folder(self, request):
-        """获取目录属性
-
-        :param request:
-        :return:
-        """
-        assert isinstance(request, StatFolderRequest)
-        return self.stat_base(request)
-
-    def create_folder(self, request):
-        """创建目录
-
-        :param request:
-        :return:
-        """
-        assert isinstance(request, CreateFolderRequest)
-        check_params_ret = self._check_params(request)
-        if check_params_ret is not None:
-            return check_params_ret
-
-        auth = cos_auth.Auth(self._cred)
-        bucket = request.get_bucket_name()
-        cos_path = request.get_cos_path()
-        expired = int(time.time()) + self._expired_period
-        sign = auth.sign_more(bucket, cos_path, expired)
-
-        http_header = dict()
-        http_header['Authorization'] = sign
-        http_header['Content-Type'] = 'application/json'
-        http_header['User-Agent'] = self._config.get_user_agent()
-
-        http_body = dict()
-        http_body['op'] = 'create'
-        http_body['biz_attr'] = request.get_biz_attr()
-
-        timeout = self._config.get_timeout()
-        return self.send_request('POST', bucket, cos_path, headers=http_header, data=json.dumps(http_body), timeout=timeout)
-
-    def list_folder(self, request):
-        """list目录
-
-        :param request:
-        :return:
-        """
-        assert isinstance(request, ListFolderRequest)
-        check_params_ret = self._check_params(request)
-        if check_params_ret is not None:
-            return check_params_ret
-
-        http_body = dict()
-        http_body['op'] = 'list'
-        http_body['num'] = request.get_num()
-
-        http_body['context'] = request.get_context()
-
-        auth = cos_auth.Auth(self._cred)
-        bucket = request.get_bucket_name()
-        list_path = request.get_cos_path() + request.get_prefix()
-        expired = int(time.time()) + self._expired_period
-        sign = auth.sign_more(bucket, list_path, expired)
-
-        http_header = dict()
-        http_header['Authorization'] = sign
-        http_header['User-Agent'] = self._config.get_user_agent()
-
-        timeout = self._config.get_timeout()
-        return self.send_request('GET', bucket, list_path, headers=http_header, params=http_body, timeout=timeout)

tools/cos/qcloud_cos/cos_params_check.py

@@ -1,225 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-import os
-import re
-
-
-class ParamCheck(object):
-    """BaseRequest基本类型的请求"""
-    def __init__(self):
-        self._err_tips = u''
-
-    def get_err_tips(self):
-        """获取错误信息
-
-        :return:
-        """
-        return self._err_tips
-
-    def check_param_unicode(self, param_name, param_value):
-        """检查参数是否是unicode
-
-        :param param_name: param_name 参数名
-        :param param_value: param_value 参数值
-        :return:
-        """
-        if param_value is None:
-            self._err_tips = param_name + ' is None!'
-            return False
-        if not isinstance(param_value, unicode):
-            self._err_tips = param_name + ' is not unicode!'
-            return False
-        return True
-
-    def check_param_int(self, param_name, param_value):
-        """检查参数是否是int
-
-        :param param_name: param_name 参数名
-        :param param_value: param_value 参数值
-        :return:
-        """
-        if param_value is None:
-            self._err_tips = param_name + ' is None!'
-            return False
-        if not isinstance(param_value, int):
-            self._err_tips = param_name + ' is not int!'
-            return False
-        return True
-
-    def check_cos_path_valid(self, cos_path, is_file_path):
-        """检查cos_path是否合法
-
-        路径必须以/开始,文件路径则不能以/结束, 目录路径必须以/结束
-
-        :param cos_path:
-        :param is_file_path:
-        :return: True for valid path, other False
-        """
-        if cos_path[0] != u'/':
-            self._err_tips = 'cos path must start with /'
-            return False
-
-        last_letter = cos_path[len(cos_path) - 1]
-        if is_file_path and last_letter == u'/':
-            self._err_tips = 'for file operation, cos_path must not end with /'
-            return False
-        elif not is_file_path and last_letter != u'/':
-            self._err_tips = 'for folder operation, cos_path must end with /'
-            return False
-        else:
-            pass
-
-        illegal_letters = ['?', '*', ':', '|', '\\', '<', '>', '"']
-        for illegal_letter in illegal_letters:
-            if cos_path.find(illegal_letter) != -1:
-                self._err_tips = 'cos path contain illegal letter %s' % illegal_letter
-                return False
-
-        pattern = re.compile(r'/(\s*)/')
-        if pattern.search(cos_path):
-            self._err_tips = 'cos path contain illegal letter / /'
-            return False
-        return True
-
-    def check_not_cos_root(self, cos_path):
-        """检查不是cos的根路径
-
-        不能对根路径操作的有 1 update 2 create 3 delete
-        :param cos_path:
-        :return:
-        """
-        if cos_path == u'/':
-            self._err_tips = 'bucket operation is not supported by sdk,'
-            ' please use cos console: https://console.qcloud.com/cos'
-            return False
-        else:
-            return True
-
-    def check_local_file_valid(self, local_path):
-        """检查本地文件有效(存在并且可读)
-
-        :param local_path:
-        :return:
-        """
-        if not os.path.exists(local_path):
-            self._err_tips = 'local_file %s not exist!' % local_path
-            return False
-        if not os.path.isfile(local_path):
-            self._err_tips = 'local_file %s is not regular file!' % local_path
-            return False
-        if not os.access(local_path, os.R_OK):
-            self._err_tips = 'local_file %s is not readable!' % local_path
-            return False
-        return True
-
-    def check_slice_size(self, slice_size):
-        """检查分片大小有效
-
-        :param slice_size:
-        :return:
-        """
-        min_size = 64 * 1024           # 512KB
-        max_size = 3 * 1024 * 1024     # 20MB
-
-        if max_size >= slice_size >= min_size:
-            return True
-        else:
-            self._err_tips = 'slice_size is invalid, only accept [%d, %d]' \
-                    % (min_size, max_size)
-            return False
-
-    def check_insert_only(self, insert_only):
-        """检查文件上传的insert_only参数
-
-        :param insert_only:
-        :return:
-        """
-        if insert_only != 1 and insert_only != 0:
-            self._err_tips = 'insert_only only support 0 and 1'
-            return False
-        else:
-            return True
-
-    def check_move_over_write(self, to_over_write):
-        """检查move的over write标志
-
-        :param to_over_write:
-        :return:
-        """
-        if to_over_write != 1 and to_over_write != 0:
-            self._err_tips = 'to_over_write only support 0 and 1'
-            return False
-        else:
-            return True
-
-    def check_file_authority(self, authority):
-        """检查文件的authority属性
-
-        合法的取值只有eInvalid, eWRPrivate, eWPrivateRPublic和空值
-        :param authority:
-        :return:
-        """
-        if authority != u''and authority != u'eInvalid' and authority != u'eWRPrivate' and authority != u'eWPrivateRPublic':
-            self._err_tips = 'file authority valid value is: eInvalid, eWRPrivate, eWPrivateRPublic'
-            return False
-        else:
-            return True
-
-    def check_x_cos_meta_dict(self, x_cos_meta_dict):
-        """检查x_cos_meta_dict, key和value都必须是UTF8编码
-
-        :param x_cos_meta_dict:
-        :return:
-        """
-        prefix_len = len('x-cos-meta-')
-        for key in x_cos_meta_dict.keys():
-            if not self.check_param_unicode('x-cos-meta-key', key):
-                return False
-            if not self.check_param_unicode('x-cos-meta-value', x_cos_meta_dict[key]):
-                return False
-            if key[0:prefix_len] != u'x-cos-meta-':
-                self._err_tips = 'x-cos-meta key must start with x-cos-meta-'
-                return False
-            if len(key) == prefix_len:
-                self._err_tips = 'x-cos-meta key must not just be x-cos-meta-'
-                return False
-            if len(x_cos_meta_dict[key]) == 0:
-                self._err_tips = 'x-cos-meta value must not be empty'
-                return False
-        return True
-
-    def check_update_flag(self, flag):
-        """检查更新文件的flag
-
-        :param flag:
-        :return:
-        """
-        if flag == 0:
-            self._err_tips = 'no any attribute to be updated!'
-            return False
-        else:
-            return True
-
-    def check_list_order(self, list_order):
-        """ 检查list folder的order
-
-        :param list_order: 合法取值0(正序), 1(逆序)
-        :return:
-        """
-        if list_order != 0 and list_order != 1:
-            self._err_tips = 'list order is invalid, please use 0(positive) or 1(reverse)!'
-            return False
-        else:
-            return True
-
-    def check_list_pattern(self, list_pattern):
-        """检查list folder的pattern
-
-        :param list_pattern: 合法取值eListBoth, eListDirOnly, eListFileOnly
-        :return:
-        """
-        if list_pattern != u'eListBoth' and list_pattern != u'eListDirOnly' and list_pattern != u'eListFileOnly':
-            self._err_tips = 'list pattern is invalid, please use eListBoth or eListDirOnly or eListFileOnly'
-            return False
-        else:
-            return True

tools/cos/qcloud_cos/cos_request.py

@@ -1,630 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
-"""the request type in tencent qcloud cos"""
-
-from cos_params_check import ParamCheck
-import collections
-
-
-class BaseRequest(object):
-    """BaseRequest基本类型的请求"""
-
-    def __init__(self, bucket_name, cos_path):
-        """ 类初始化
-
-        :param bucket_name: bucket的名称
-        :param cos_path: cos的绝对路径, 即从bucket下的根/开始
-        """
-        self._bucket_name = bucket_name.strip()
-        self._cos_path = cos_path.strip()
-        self._param_check = ParamCheck()
-
-    def set_bucket_name(self, bucket_name=u''):
-        """设置bucket_name
-
-        :param bucket_name:
-        :return:
-        """
-        self._bucket_name = bucket_name.strip()
-
-    def get_bucket_name(self):
-        """获取bucket_name
-
-        :return:
-        """
-        return self._bucket_name
-
-    def set_cos_path(self, cos_path=u''):
-        """设置cos_path
-
-        :param cos_path:
-        :return:
-        """
-        self._cos_path = cos_path.strip()
-
-    def get_cos_path(self):
-        """获取cos_path
-
-        :return:
-        """
-        return self._cos_path
-
-    def get_err_tips(self):
-        """获取错误信息
-
-        :return:
-        """
-        return self._param_check.get_err_tips()
-
-    def check_params_valid(self):
-        """检查参数是否合法
-
-        :return:
-        """
-        if not self._param_check.check_param_unicode('bucket', self._bucket_name):
-            return False
-        return self._param_check.check_param_unicode('cos_path', self._cos_path)
-
-
-class CreateFolderRequest(BaseRequest):
-    """CreateFolderRequest  创建目录类型的请求"""
-
-    def __init__(self, bucket_name, cos_path, biz_attr=u''):
-        """
-
-        :param bucket_name: bucket的名称
-        :param cos_path: cos的绝对路径, 从bucket根/开始
-        :param biz_attr: 目录的属性
-        """
-        super(CreateFolderRequest, self).__init__(bucket_name, cos_path)
-        self._biz_attr = biz_attr
-
-    def set_biz_attr(self, biz_attr):
-        """设置biz_attr
-
-        :param biz_attr:
-        :return:
-        """
-        self._biz_attr = biz_attr
-
-    def get_biz_attr(self):
-        """ 获取biz_attr
-
-        :return:
-        """
-        return self._biz_attr
-
-    def check_params_valid(self):
-        """检查参数是否合法
-
-        :return:
-        """
-        if not super(CreateFolderRequest, self).check_params_valid():
-            return False
-        if not self._param_check.check_param_unicode('biz_attr', self._biz_attr):
-            return False
-        if not self._param_check.check_cos_path_valid(self._cos_path, is_file_path=False):
-            return False
-        return self._param_check.check_not_cos_root(self._cos_path)
-
-
-class UploadFileRequest(BaseRequest):
-    """
-    UploadFileRequest  单文件上传请求
-    """
-
-    def __init__(self, bucket_name, cos_path, local_path, biz_attr=u'', insert_only=1):
-        """
-
-        :param bucket_name:  bucket的名称
-        :param cos_path: cos的绝对路径(目的路径), 从bucket根/开始
-        :param local_path: 上传的本地文件路径(源路径)
-        :param biz_attr: 文件的属性
-        :param insert_only: 是否覆盖写, 0覆盖, 1不覆盖,返回错误
-        """
-        super(UploadFileRequest, self).__init__(bucket_name, cos_path)
-        self._local_path = local_path.strip()
-        self._biz_attr = biz_attr
-        self._insert_only = insert_only
-
-    def set_local_path(self, local_path):
-        """设置local_path
-
-        :param local_path:
-        :return:
-        """
-        self._local_path = local_path.strip()
-
-    def get_local_path(self):
-        """获取local_path
-
-        :return:
-        """
-        return self._local_path
-
-    def set_biz_attr(self, biz_attr):
-        """设置biz_attr
-
-        :param biz_attr:
-        :return:
-        """
-        self._biz_attr = biz_attr
-
-    def get_biz_attr(self):
-        """获取biz_attr
-
-        :return:
-        """
-        return self._biz_attr
-
-    def set_insert_only(self, insert_only):
-        """设置insert_only，0表示如果文件存在, 则覆盖
-
-        :param insert_only:
-        :return:
-        """
-        self._insert_only = insert_only
-
-    def get_insert_only(self):
-        """获取insert_only
-
-        :return:
-        """
-        return self._insert_only
-
-    def check_params_valid(self):
-        """检查参数是否有效
-
-        :return:
-        """
-        if not super(UploadFileRequest, self).check_params_valid():
-            return False
-        if not self._param_check.check_cos_path_valid(self._cos_path, is_file_path=True):
-            return False
-        if not self._param_check.check_param_unicode('biz_attr', self._biz_attr):
-            return False
-        if not self._param_check.check_param_unicode('local_path', self._local_path):
-            return False
-        if not self._param_check.check_local_file_valid(self._local_path):
-            return False
-        if not self._param_check.check_param_int('insert_only', self._insert_only):
-            return False
-        return self._param_check.check_insert_only(self._insert_only)
-
-
-class UploadSliceFileRequest(UploadFileRequest):
-    """
-    UploadSliceFileRequest  分片文件上传请求
-    """
-
-    def __init__(self, bucket_name, cos_path, local_path, slice_size=1024*1024, biz_attr=u'', enable_sha1=False, max_con=1):
-        """
-
-        :param bucket_name: bucket的名称
-        :param cos_path: cos的绝对路径(目的路径), 从bucket根/开始
-        :param local_path: 上传的本地文件路径(源路径)
-        :param slice_size: 文件的属性
-        :param biz_attr: 分片大小(字节, 默认1MB)
-        :param enable_sha1: 是否启用sha1校验
-        """
-        super(UploadSliceFileRequest, self).__init__(bucket_name, cos_path, local_path, biz_attr)
-        self._slice_size = slice_size
-        self._enable_sha1 = enable_sha1
-        self._max_con = max_con
-
-    @property
-    def enable_sha1(self):
-        return self._enable_sha1
-
-    @enable_sha1.setter
-    def enable_sha1(self, val):
-        if val in (True, False):
-            self._enable_sha1 = val
-        else:
-            raise ValueError("enable_sha1 should be True/False")
-
-    def set_slice_size(self, slice_size):
-        """设置分片大小
-
-        :param slice_size:
-        :return:
-        """
-        self._slice_size = slice_size
-
-    def get_slice_size(self):
-        """获取分片大小
-
-        :return:
-        """
-        return self._slice_size
-
-    def check_params_valid(self):
-        """检查参数是否有效
-
-        :return:
-        """
-        if not super(UploadSliceFileRequest, self).check_params_valid():
-            return False
-
-        if self._enable_sha1 and self._slice_size != 1024*1024:
-            self._param_check._err_tips = 'slice_size is invalid, slice must be 1MB with enable_sha1'
-            return False
-
-        return self._param_check.check_slice_size(self._slice_size)
-
-
-class UpdateFolderRequest(BaseRequest):
-    """UpdateFolderRequest 更新目录请求"""
-
-    def __init__(self, bucket_name, cos_path, biz_attr=u''):
-        """
-
-        :param bucket_name: bucket name
-        :param cos_path: the path on cos
-        :param biz_attr: biz attributes
-        """
-        super(UpdateFolderRequest, self).__init__(bucket_name, cos_path)
-        self._biz_attr = biz_attr
-
-    def set_biz_attr(self, biz_attr):
-        """设置biz_attr
-
-        :param biz_attr:
-        :return:
-        """
-        self._biz_attr = biz_attr
-
-    def get_biz_attr(self):
-        """获取biz_attr
-
-        :return:
-        """
-        return self._biz_attr
-
-    def check_params_valid(self):
-        """检查参数是否有效
-
-        :return:
-        """
-        if not super(UpdateFolderRequest, self).check_params_valid():
-            return False
-        if not self._param_check.check_cos_path_valid(self._cos_path, is_file_path=False):
-            return False
-        if not self._param_check.check_not_cos_root(self._cos_path):
-            return False
-        return self._param_check.check_param_unicode('biz_attr', self._biz_attr)
-
-
-class UpdateFileRequest(BaseRequest):
-    """UpdateFileRequest 更新文件请求 """
-
-    def __init__(self, bucket_name, cos_path):
-        """ 初始化类
-
-            biz_attr:     要更新的文件的属性
-            authority:              文件权限:
-                             eInvalid(继承bucket),
-                             eWRPrivate(私有读写),
-                             eWPrivateRPublic(私有写, 公有读)
-            customer_header:        用户自定义的HTTP请求头,包括以下成员
-            cache_control:          文件的缓存机制,参见HTTP的Cache-Control
-            content_type:           文件的MIME信息,参见HTTP的Content-Type
-            content_disposition:    MIME协议的扩展,参见HTTP的Content-Disposition
-            content_language:       文件的语言, 参见HTTP的Content-Language
-            content_encoding:       body的编码, 参见HTTP的Content-Encoding
-            _x_cos_meta_dict:       用户自定义的属性, key是以x-cos-meta-开头,value为属性值
-
-        :param bucket_name: bucket的名称
-        :param cos_path: cos的绝对路径, 从bucket根/开始
-        """
-        super(UpdateFileRequest, self).__init__(bucket_name, cos_path)
-        self._biz_attr = None
-        self._custom_headers = {}
-        self._authority = None
-        self._cache_control = None
-        self._content_type = None
-        self._content_disposition = None
-        self._content_language = None
-        self._content_encoding = None
-        self._x_cos_meta_dict = dict()
-
-    def set_biz_attr(self, biz_attr):
-        """设置biz_attr"""
-        self._biz_attr = biz_attr
-
-    def get_biz_attr(self):
-        """获取biz_attr"""
-        return self._biz_attr
-
-    # 设置authority, 合法取值如下所示
-    # eInvalid(继承bucket),
-    # eWRPrivate(私有读写),
-    # eWPrivateRPublic(私有写, 公有读)
-    def set_authority(self, authority):
-        """设置authority,
-
-        合法取值:eInvalid(继承bucket),eWRPrivate(私有读写),eWPrivateRPublic(私有写, 公有读)
-        :param authority:
-        :return:
-        """
-        self._authority = authority
-
-    def get_authority(self):
-        """获取authority"""
-        return self._authority
-
-    def set_cache_control(self, cache_control):
-        """设置缓存机制Cache-Control"""
-        self._cache_control = cache_control
-        self._custom_headers[u'Cache-Control'] = cache_control
-
-    def set_content_type(self, content_type):
-        """设置Content-Type"""
-        self._content_type = content_type
-        self._custom_headers['Content-Type'] = content_type
-
-    def set_content_disposition(self, content_disposition):
-        """设置Content-Disposition"""
-        self._content_disposition = content_disposition
-        self._custom_headers['Content-Disposition'] = content_disposition
-
-    def set_content_language(self, content_language):
-        """设置Content-Language"""
-        self._content_language = content_language
-        self._custom_headers['Content-Language'] = content_language
-
-    def set_content_encoding(self, content_encoding):
-        """设置Content-Encoding"""
-        self._content_encoding = content_encoding
-        self._custom_headers['Content-Encoding'] = content_encoding
-
-    def set_x_cos_meta(self, key, value):
-        """设置自定义的x-cos-meta
-
-        key以x-cos-meta-开头,例如自定义key为u'x-cos-meta-len', value为u'1024'
-        :param key:
-        :param value:
-        :return:
-        """
-        self._x_cos_meta_dict[key] = value
-        self._custom_headers[key] = value
-
-    def _convert_dict(self, data):
-        """convert a dict's keys & values from `unicode` to `str`
-
-        :param data:
-        :return:
-        """
-        if isinstance(data, basestring):
-            return str(data)
-        elif isinstance(data, collections.Mapping):
-            return dict(map(self._convert_dict, data.iteritems()))
-        elif isinstance(data, collections.Iterable):
-            return type(data)(map(self._convert_dict, data))
-        else:
-            return data
-
-    def get_custom_headers(self):
-        """ 获取自定义的HTTP头"""
-        return self._convert_dict(self._custom_headers)
-
-    def check_params_valid(self):
-        """ 检查参数是否合法"""
-        if not super(UpdateFileRequest, self).check_params_valid():
-            return False
-
-        if not self._param_check.check_cos_path_valid(self._cos_path, is_file_path=True):
-            return False
-
-        if self._biz_attr is not None:
-            if not self._param_check.check_param_unicode('biz_attr', self._biz_attr):
-                return False
-
-        if self._authority is not None:
-            if not self._param_check.check_param_unicode('authority', self._authority):
-                return False
-
-        if self._authority is not None:
-            if not self._param_check.check_file_authority(self._authority):
-                return False
-
-        if self._cache_control is not None:
-            if not self._param_check.check_param_unicode('cache_control', self._cache_control):
-                return False
-
-        if self._content_type is not None:
-            if not self._param_check.check_param_unicode('content_type', self._content_type):
-                return False
-
-        if self._content_disposition is not None:
-            if not self._param_check.check_param_unicode('content_disposition', self._content_disposition):
-                return False
-
-        if self._content_language is not None:
-            if not self._param_check.check_param_unicode('content_language', self._content_language):
-                return False
-
-        if self._content_encoding is not None:
-            if not self._param_check.check_param_unicode('content_encoding', self._content_encoding):
-                return False
-
-        return self._param_check.check_x_cos_meta_dict(self._x_cos_meta_dict)
-
-
-class StatFileRequest(BaseRequest):
-    """StatRequest 获取文件属性请求"""
-
-    def __init__(self, bucket_name, cos_path):
-        """
-        :param bucket_name: bucket的名称
-        :param cos_path: cos的文件路径, 从bucket根/开始, 不以/结束
-        """
-        super(StatFileRequest, self).__init__(bucket_name, cos_path)
-
-    def check_params_valid(self):
-        """检查参数是否合法"""
-        if not super(StatFileRequest, self).check_params_valid():
-            return False
-        return self._param_check.check_cos_path_valid(self._cos_path, is_file_path=True)
-
-
-class StatFolderRequest(BaseRequest):
-    """StatRequest 获取目录属性请求 """
-
-    def __init__(self, bucket_name, cos_path):
-        """
-
-        :param bucket_name: bucket的名称
-        :param cos_path: cos的目录路径, 从bucket根/开始, 以/结束
-        """
-        super(StatFolderRequest, self).__init__(bucket_name, cos_path)
-
-    def check_params_valid(self):
-        """检查参数是否合法"""
-        if not super(StatFolderRequest, self).check_params_valid():
-            return False
-        return self._param_check.check_cos_path_valid(self._cos_path, is_file_path=False)
-
-
-class DelFileRequest(BaseRequest):
-    """ DelFileRequest 删除文件请求 """
-
-    def __init__(self, bucket_name, cos_path):
-        """
-
-        :param bucket_name: bucket的名称
-        :param cos_path: cos的文件路径, 从bucket根/开始, 不以/结束
-        """
-        super(DelFileRequest, self).__init__(bucket_name, cos_path)
-
-    def check_params_valid(self):
-        """检查参数是否合法"""
-        if not super(DelFileRequest, self).check_params_valid():
-            return False
-        return self._param_check.check_cos_path_valid(self._cos_path, is_file_path=True)
-
-
-class DelFolderRequest(BaseRequest):
-    """DelFolderRequest 删除目录请求"""
-
-    def __init__(self, bucket_name, cos_path):
-        """
-
-        :param bucket_name: bucket的名称
-        :param cos_path: cos的目录路径, 从bucket根/开始, 以/结束
-        """
-        super(DelFolderRequest, self).__init__(bucket_name, cos_path)
-
-    def check_params_valid(self):
-        """ 检查参数合法"""
-        if not super(DelFolderRequest, self).check_params_valid():
-            return False
-        if not self._param_check.check_cos_path_valid(self._cos_path, is_file_path=False):
-            return False
-        return self._param_check.check_not_cos_root(self._cos_path)
-
-
-class ListFolderRequest(BaseRequest):
-    """ListFolderRequest 获取目录列表的请求"""
-
-    def __init__(self, bucket_name, cos_path, num=199, prefix=u'', context=u''):
-        """
-        :param bucket_name: bucket的名称
-        :param cos_path: cos的绝对路径, 从bucket根/开始
-        :param num: 搜索数量
-        :param prefix: 搜索前缀
-        :param context: 搜索上下文
-        """
-        super(ListFolderRequest, self).__init__(bucket_name, cos_path)
-        self._num = num
-        self._prefix = prefix
-        self._context = context
-
-    def set_num(self, num):
-        """设置List数量
-
-        :param num:
-        :return:
-        """
-        self._num = num
-
-    def get_num(self):
-        """获取List数量
-
-        :return:
-        """
-        """
-
-        :return:
-        """
-        return self._num
-
-    def set_prefix(self, prefix):
-        """设置前缀"""
-        self._prefix = prefix
-
-    def get_prefix(self):
-        """获取前缀"""
-        return self._prefix
-
-    def set_context(self, context):
-        """设置搜索上下文"""
-        self._context = context
-
-    def get_context(self):
-        """获取搜索上下文"""
-        return self._context
-
-    def check_params_valid(self):
-        """检查参数是否有效"""
-        if not super(ListFolderRequest, self).check_params_valid():
-            return False
-        if not self._param_check.check_cos_path_valid(self._cos_path, is_file_path=False):
-            return False
-        if not self._param_check.check_param_unicode('prefix', self._prefix):
-            return False
-        return self._param_check.check_param_unicode('context', self._context)
-
-
-class DownloadFileRequest(BaseRequest):
-    def __init__(self, bucket_name, cos_path, local_filename, range_start=None, range_end=None, *args, **kwargs):
-        super(DownloadFileRequest, self).__init__(bucket_name, cos_path)
-
-        self._local_filename = local_filename
-        self._range_start = range_start
-        self._range_end = range_end
-
-        self._custom_headers = None
-        if 'headers' in kwargs:
-            self._custom_headers = kwargs['headers']
-
-    def check_params_valid(self):
-        if not super(DownloadFileRequest, self).check_params_valid():
-            return False
-
-        from os import path
-        if path.exists(self._local_filename):
-            return False
-
-
-class MoveFileRequest(BaseRequest):
-
-    def __init__(self, bucket_name, cos_path, dest_path, overwrite=False):
-        super(MoveFileRequest, self).__init__(bucket_name, cos_path)
-        self._dest_path = dest_path
-        if isinstance(overwrite, bool):
-            if overwrite:
-                self._overwrite = 1
-            else:
-                self._overwrite = 0
-        else:
-            raise ValueError("overwrite must be an instance of Boolean")
-
-    @property
-    def dest_path(self):
-        return self._dest_path
-
-    @property
-    def overwrite(self):
-        return self._overwrite

tools/cos/qcloud_cos/threadpool.py

@@ -1,95 +0,0 @@
-# -*- coding: utf-8 -*-
-
-from threading import Thread
-from logging import getLogger
-from Queue import Queue
-from threading import Lock
-
-logger = getLogger(__name__)
-
-
-class WorkerThread(Thread):
-    def __init__(self, task_queue, *args, **kwargs):
-        super(WorkerThread, self).__init__(*args, **kwargs)
-
-        self._task_queue = task_queue
-        self._succ_task_num = 0
-        self._fail_task_num = 0
-        self._ret = list()
-
-    def run(self):
-
-        while True:
-            func, args, kwargs = self._task_queue.get()
-
-            try:
-                ret = func(*args, **kwargs)
-                self._succ_task_num += 1
-                self._ret.append(ret)
-
-            except Exception as e:
-                logger.warn(str(e))
-                self._fail_task_num += 1
-                self._ret.append(e)
-            finally:
-                self._task_queue.task_done()
-
-    def get_result(self):
-            return self._succ_task_num, self._fail_task_num, self._ret
-
-
-class SimpleThreadPool:
-
-    def __init__(self, num_threads=5):
-        self._num_threads = num_threads
-        self._queue = Queue()
-        self._lock = Lock()
-        self._active = False
-        self._workers = list()
-        self._finished = False
-
-    def add_task(self, func, *args, **kwargs):
-        if not self._active:
-            with self._lock:
-                if not self._active:
-                    self._active = True
-
-                    for i in range(self._num_threads):
-                        w = WorkerThread(self._queue)
-                        self._workers.append(w)
-                        w.start()
-
-        self._queue.put((func, args, kwargs))
-
-    def wait_completion(self):
-        self._queue.join()
-        self._finished = True
-
-    def get_result(self):
-        assert self._finished
-        detail = [worker.get_result() for worker in self._workers]
-        succ_all = all([tp[1] == 0 for tp in detail])
-        return {'success_all': succ_all, 'detail': detail}
-
-
-if __name__ == '__main__':
-
-    pool = SimpleThreadPool(2)
-
-    def task_sleep(x):
-        from time import sleep
-        sleep(x)
-        return 'hello, sleep %d seconds' % x
-
-    def raise_exception():
-        raise ValueError("Pa! Exception!")
-
-    pool.add_task(task_sleep, 5)
-    pool.add_task(task_sleep, 2)
-    pool.add_task(task_sleep, 3)
-    pool.add_task(raise_exception)
-    pool.add_task(raise_exception)
-
-    pool.wait_completion()
-    print pool.get_result()
-    # [(1, 0, ['hello, sleep 5 seconds']), (2, 1, ['hello, sleep 2 seconds', 'hello, sleep 3 seconds', ValueError('Pa! Exception!',)])]

tools/cos/threadpool.py

@@ -1,423 +0,0 @@
-# -*- coding: UTF-8 -*-
-"""Easy to use object-oriented thread pool framework.
-
-A thread pool is an object that maintains a pool of worker threads to perform
-time consuming operations in parallel. It assigns jobs to the threads
-by putting them in a work request queue, where they are picked up by the
-next available thread. This then performs the requested operation in the
-background and puts the results in another queue.
-
-The thread pool object can then collect the results from all threads from
-this queue as soon as they become available or after all threads have
-finished their work. It's also possible, to define callbacks to handle
-each result as it comes in.
-
-The basic concept and some code was taken from the book "Python in a Nutshell,
-2nd edition" by Alex Martelli, O'Reilly 2006, ISBN 0-596-10046-9, from section
-14.5 "Threaded Program Architecture". I wrapped the main program logic in the
-ThreadPool class, added the WorkRequest class and the callback system and
-tweaked the code here and there. Kudos also to Florent Aide for the exception
-handling mechanism.
-
-Basic usage::
-
-    >>> pool = ThreadPool(poolsize)
-    >>> requests = makeRequests(some_callable, list_of_args, callback)
-    >>> [pool.putRequest(req) for req in requests]
-    >>> pool.wait()
-
-See the end of the module code for a brief, annotated usage example.
-
-Website : http://chrisarndt.de/projects/threadpool/
-
-"""
-__docformat__ = "restructuredtext en"
-
-__all__ = [
-    'makeRequests',
-    'NoResultsPending',
-    'NoWorkersAvailable',
-    'ThreadPool',
-    'WorkRequest',
-    'WorkerThread'
-]
-
-__author__ = "Christopher Arndt"
-__version__ = '1.3.2'
-__license__ = "MIT license"
-
-
-# standard library modules
-import sys
-import threading
-import traceback
-
-try:
-    import Queue            # Python 2
-except ImportError:
-    import queue as Queue   # Python 3
-
-
-# exceptions
-class NoResultsPending(Exception):
-    """All work requests have been processed."""
-    pass
-
-class NoWorkersAvailable(Exception):
-    """No worker threads available to process remaining requests."""
-    pass
-
-
-# internal module helper functions
-def _handle_thread_exception(request, exc_info):
-    """Default exception handler callback function.
-
-    This just prints the exception info via ``traceback.print_exception``.
-
-    """
-    traceback.print_exception(*exc_info)
-
-
-# utility functions
-def makeRequests(callable_, args_list, callback=None,
-        exc_callback=_handle_thread_exception):
-    """Create several work requests for same callable with different arguments.
-
-    Convenience function for creating several work requests for the same
-    callable where each invocation of the callable receives different values
-    for its arguments.
-
-    ``args_list`` contains the parameters for each invocation of callable.
-    Each item in ``args_list`` should be either a 2-item tuple of the list of
-    positional arguments and a dictionary of keyword arguments or a single,
-    non-tuple argument.
-
-    See docstring for ``WorkRequest`` for info on ``callback`` and
-    ``exc_callback``.
-
-    """
-    requests = []
-    for item in args_list:
-        if isinstance(item, tuple):
-            requests.append(
-                WorkRequest(callable_, item[0], item[1], callback=callback,
-                    exc_callback=exc_callback)
-            )
-        else:
-            requests.append(
-                WorkRequest(callable_, [item], None, callback=callback,
-                    exc_callback=exc_callback)
-            )
-    return requests
-
-
-# classes
-class WorkerThread(threading.Thread):
-    """Background thread connected to the requests/results queues.
-
-    A worker thread sits in the background and picks up work requests from
-    one queue and puts the results in another until it is dismissed.
-
-    """
-
-    def __init__(self, requests_queue, results_queue, poll_timeout=5, **kwds):
-        """Set up thread in daemonic mode and start it immediatedly.
-
-        ``requests_queue`` and ``results_queue`` are instances of
-        ``Queue.Queue`` passed by the ``ThreadPool`` class when it creates a
-        new worker thread.
-
-        """
-        threading.Thread.__init__(self, **kwds)
-        self.setDaemon(1)
-        self._requests_queue = requests_queue
-        self._results_queue = results_queue
-        self._poll_timeout = poll_timeout
-        self._dismissed = threading.Event()
-        self.start()
-
-    def run(self):
-        """Repeatedly process the job queue until told to exit."""
-        while True:
-            if self._dismissed.isSet():
-                # we are dismissed, break out of loop
-                break
-            # get next work request. If we don't get a new request from the
-            # queue after self._poll_timout seconds, we jump to the start of
-            # the while loop again, to give the thread a chance to exit.
-            try:
-                request = self._requests_queue.get(True, self._poll_timeout)
-            #except Queue.Empty:
-            except:
-                continue
-            else:
-                if self._dismissed.isSet():
-                    # we are dismissed, put back request in queue and exit loop
-                    self._requests_queue.put(request)
-                    break
-                try:
-                    result = request.callable(*request.args, **request.kwds)
-                    self._results_queue.put((request, result))
-                except:
-                    request.exception = True
-                    self._results_queue.put((request, sys.exc_info()))
-
-    def dismiss(self):
-        """Sets a flag to tell the thread to exit when done with current job.
-        """
-        self._dismissed.set()
-
-
-class WorkRequest:
-    """A request to execute a callable for putting in the request queue later.
-
-    See the module function ``makeRequests`` for the common case
-    where you want to build several ``WorkRequest`` objects for the same
-    callable but with different arguments for each call.
-
-    """
-
-    def __init__(self, callable_, args=None, kwds=None, requestID=None,
-            callback=None, exc_callback=_handle_thread_exception):
-        """Create a work request for a callable and attach callbacks.
-
-        A work request consists of the a callable to be executed by a
-        worker thread, a list of positional arguments, a dictionary
-        of keyword arguments.
-
-        A ``callback`` function can be specified, that is called when the
-        results of the request are picked up from the result queue. It must
-        accept two anonymous arguments, the ``WorkRequest`` object and the
-        results of the callable, in that order. If you want to pass additional
-        information to the callback, just stick it on the request object.
-
-        You can also give custom callback for when an exception occurs with
-        the ``exc_callback`` keyword parameter. It should also accept two
-        anonymous arguments, the ``WorkRequest`` and a tuple with the exception
-        details as returned by ``sys.exc_info()``. The default implementation
-        of this callback just prints the exception info via
-        ``traceback.print_exception``. If you want no exception handler
-        callback, just pass in ``None``.
-
-        ``requestID``, if given, must be hashable since it is used by
-        ``ThreadPool`` object to store the results of that work request in a
-        dictionary. It defaults to the return value of ``id(self)``.
-
-        """
-        if requestID is None:
-            self.requestID = id(self)
-        else:
-            try:
-                self.requestID = hash(requestID)
-            except TypeError:
-                raise TypeError("requestID must be hashable.")
-        self.exception = False
-        self.callback = callback
-        self.exc_callback = exc_callback
-        self.callable = callable_
-        self.args = args or []
-        self.kwds = kwds or {}
-
-    def __str__(self):
-        return "<WorkRequest id=%s args=%r kwargs=%r exception=%s>" % \
-            (self.requestID, self.args, self.kwds, self.exception)
-
-class ThreadPool:
-    """A thread pool, distributing work requests and collecting results.
-
-    See the module docstring for more information.
-
-    """
-
-    def __init__(self, num_workers, q_size=0, resq_size=0, poll_timeout=5):
-        """Set up the thread pool and start num_workers worker threads.
-
-        ``num_workers`` is the number of worker threads to start initially.
-
-        If ``q_size > 0`` the size of the work *request queue* is limited and
-        the thread pool blocks when the queue is full and it tries to put
-        more work requests in it (see ``putRequest`` method), unless you also
-        use a positive ``timeout`` value for ``putRequest``.
-
-        If ``resq_size > 0`` the size of the *results queue* is limited and the
-        worker threads will block when the queue is full and they try to put
-        new results in it.
-
-        .. warning:
-            If you set both ``q_size`` and ``resq_size`` to ``!= 0`` there is
-            the possibilty of a deadlock, when the results queue is not pulled
-            regularly and too many jobs are put in the work requests queue.
-            To prevent this, always set ``timeout > 0`` when calling
-            ``ThreadPool.putRequest()`` and catch ``Queue.Full`` exceptions.
-
-        """
-        self._requests_queue = Queue.Queue(q_size)
-        self._results_queue = Queue.Queue(resq_size)
-        self.workers = []
-        self.dismissedWorkers = []
-        self.workRequests = {}
-        self.createWorkers(num_workers, poll_timeout)
-
-    def createWorkers(self, num_workers, poll_timeout=5):
-        """Add num_workers worker threads to the pool.
-
-        ``poll_timout`` sets the interval in seconds (int or float) for how
-        ofte threads should check whether they are dismissed, while waiting for
-        requests.
-
-        """
-        for i in range(num_workers):
-            self.workers.append(WorkerThread(self._requests_queue,
-                self._results_queue, poll_timeout=poll_timeout))
-
-    def dismissWorkers(self, num_workers, do_join=False):
-        """Tell num_workers worker threads to quit after their current task."""
-        dismiss_list = []
-        for i in range(min(num_workers, len(self.workers))):
-            worker = self.workers.pop()
-            worker.dismiss()
-            dismiss_list.append(worker)
-
-        if do_join:
-            for worker in dismiss_list:
-                worker.join()
-        else:
-            self.dismissedWorkers.extend(dismiss_list)
-
-    def joinAllDismissedWorkers(self):
-        """Perform Thread.join() on all worker threads that have been dismissed.
-        """
-        for worker in self.dismissedWorkers:
-            worker.join()
-        self.dismissedWorkers = []
-
-    def putRequest(self, request, block=True, timeout=None):
-        """Put work request into work queue and save its id for later."""
-        assert isinstance(request, WorkRequest)
-        # don't reuse old work requests
-        assert not getattr(request, 'exception', None)
-        self._requests_queue.put(request, block, timeout)
-        self.workRequests[request.requestID] = request
-
-    def poll(self, block=False):
-        """Process any new results in the queue."""
-        while True:
-            # still results pending?
-            if not self.workRequests:
-                raise NoResultsPending
-            # are there still workers to process remaining requests?
-            elif block and not self.workers:
-                raise NoWorkersAvailable
-            try:
-                # get back next results
-                request, result = self._results_queue.get(block=block)
-                # has an exception occured?
-                if request.exception and request.exc_callback:
-                    request.exc_callback(request, result)
-                # hand results to callback, if any
-                if request.callback and not \
-                       (request.exception and request.exc_callback):
-                    request.callback(request, result)
-                del self.workRequests[request.requestID]
-            #except Queue.Empty:
-            except:
-                break
-
-    def wait(self):
-        """Wait for results, blocking until all have arrived."""
-        while 1:
-            try:
-                self.poll(True)
-            except NoResultsPending:
-                break
-
-
-################
-# USAGE EXAMPLE
-################
-
-if __name__ == '__main__':
-    import random
-    import time
-
-    # the work the threads will have to do (rather trivial in our example)
-    def do_something(data):
-        time.sleep(random.randint(1,5))
-        result = round(random.random() * data, 5)
-        # just to show off, we throw an exception once in a while
-        if result > 5:
-            raise RuntimeError("Something extraordinary happened!")
-        return result
-
-    # this will be called each time a result is available
-    def print_result(request, result):
-        print("**** Result from request #%s: %r" % (request.requestID, result))
-
-    # this will be called when an exception occurs within a thread
-    # this example exception handler does little more than the default handler
-    def handle_exception(request, exc_info):
-        if not isinstance(exc_info, tuple):
-            # Something is seriously wrong...
-            print(request)
-            print(exc_info)
-            raise SystemExit
-        print("**** Exception occured in request #%s: %s" % \
-          (request.requestID, exc_info))
-
-    # assemble the arguments for each job to a list...
-    data = [random.randint(1,10) for i in range(20)]
-    # ... and build a WorkRequest object for each item in data
-    requests = makeRequests(do_something, data, print_result, handle_exception)
-    # to use the default exception handler, uncomment next line and comment out
-    # the preceding one.
-    #requests = makeRequests(do_something, data, print_result)
-
-    # or the other form of args_lists accepted by makeRequests: ((,), {})
-    data = [((random.randint(1,10),), {}) for i in range(20)]
-    requests.extend(
-        makeRequests(do_something, data, print_result, handle_exception)
-        #makeRequests(do_something, data, print_result)
-        # to use the default exception handler, uncomment next line and comment
-        # out the preceding one.
-    )
-
-    # we create a pool of 3 worker threads
-    print("Creating thread pool with 3 worker threads.")
-    main = ThreadPool(3)
-
-    # then we put the work requests in the queue...
-    for req in requests:
-        main.putRequest(req)
-        print("Work request #%s added." % req.requestID)
-    # or shorter:
-    # [main.putRequest(req) for req in requests]
-
-    # ...and wait for the results to arrive in the result queue
-    # by using ThreadPool.wait(). This would block until results for
-    # all work requests have arrived:
-    # main.wait()
-
-    # instead we can poll for results while doing something else:
-    i = 0
-    while True:
-        try:
-            time.sleep(0.5)
-            main.poll()
-            print("Main thread working...")
-            print("(active worker threads: %i)" % (threading.activeCount()-1, ))
-            if i == 10:
-                print("**** Adding 3 more worker threads...")
-                main.createWorkers(3)
-            if i == 20:
-                print("**** Dismissing 2 worker threads...")
-                main.dismissWorkers(2)
-            i += 1
-        except KeyboardInterrupt:
-            print("**** Interrupted!")
-            break
-        except NoResultsPending:
-            print("**** No pending results.")
-            break
-    if main.dismissedWorkers:
-        print("Joining all dismissed worker threads...")
-        main.joinAllDismissedWorkers()

tools/coscmd

@@ -1,508 +0,0 @@
-#!/usr/bin/env python
-#coding:utf-8
-
-import sys,os
-import datetime
-import random
-import threading
-import time
-import datetime
-import logging
-import ConfigParser
-from optparse import OptionParser
-from logging.handlers import RotatingFileHandler
-from time import strftime, localtime
-from time import sleep
-from datetime import date
-from datetime import timedelta
-from cos import CosClient
-from cos import UploadFileRequest
-from cos import CreateFolderRequest
-from cos import DelFileRequest
-from cos import DelFolderRequest
-from cos import ListFolderRequest
-from cos import threadpool 
-
-MAX_RETRY_TIMES = 3
-LOG_SAVE_EVERY_NUM = 1024
-ONE_TASK_DEL_FILE_NUMS = 50
-log_level = 1
-log_file_name = "del_file.log"
-dir_thread_num = 2
-file_thread_num = 5
-log_out_to_screen = 1
-delete_folder_fail_exist = 0
-
-CONFIGFILE = "%s/.coscredentials" % os.path.expanduser('~')
-CONFIGSECTION = 'COSCredentials'
-
-HAS_FORK = hasattr(os, 'fork')
-
-HELP = \
-'''coscmd:
-    config         --appid=[appid] --id=[secret_id] --key=[secret_key] --region=[region] --bucket=[bucket] 
-    ls             cosdir
-    mkdir          dirname
-    put            localfile  cosdir 
-    rm(delete,del) object
-    '''
-
-CMD_LIST = {}
-def cmd_configure(args, options):
-    if options.appid is None or options.secret_id is None or options.secret_key is None  or options.region is None or options.bucket is None:
-        print("%s miss parameters, use --appid=[appid] --id=[secret_id] --key=[secret_key] --region=[region] --bucket=[bucket] to specify appid/id/key/region/bucket pair" % args[0])
-        sys.exit(-1)
-    config = ConfigParser.RawConfigParser()
-    config.add_section(CONFIGSECTION)
-    config.set(CONFIGSECTION, 'appid', options.appid)
-    config.set(CONFIGSECTION, 'secret_id', options.secret_id)
-    config.set(CONFIGSECTION, 'secret_key', options.secret_key)
-    if options.region in ['sh','gz','tj','cd','sgp']:
-        config.set(CONFIGSECTION, 'region', options.region)
-    else:
-        print("input region error, setup use : --region={sh,gz,tj,cd,sgp}")
-        sys.exit(-1)
-    config.set(CONFIGSECTION, 'bucket', options.bucket)
-    cfgfile = open(CONFIGFILE, 'w+')
-    config.write(cfgfile)
-    print("Your configuration is saved into %s ." % CONFIGFILE)
-    cfgfile.close()
-    import stat
-    os.chmod(CONFIGFILE, stat.S_IREAD | stat.S_IWRITE)
-
-def cmd_loadconfigure():
-    config = ConfigParser.ConfigParser()
-    config.read(CONFIGFILE)
-    global appid
-    global secret_id
-    global secret_key
-    global region 
-    global bucket
-    appid = int(config.get(CONFIGSECTION, 'appid'))
-    secret_id = config.get(CONFIGSECTION, 'secret_id').decode('utf-8')
-    secret_key = config.get(CONFIGSECTION, 'secret_key').decode('utf-8')
-    region = config.get(CONFIGSECTION, 'region')
-    bucket = config.get(CONFIGSECTION, 'bucket').decode('utf-8')
-    if len(secret_id) == 0 or len(secret_key) == 0 or len(region) == 0 or len(bucket) == 0:
-        print("can't get appid/secret_id/secret_key/region/bucket, setup use : config --appid=[appid] --id=[secret_id] --key=[secret_key] --region=[region] --bucket=[bucket]")
-        sys.exit(1)
-
-def cmd_lsdir(COSDIR):
-    cosdir = COSDIR.decode('utf-8')
-    request = ListFolderRequest(bucket, cosdir)
-    list_folder_ret = cos_client.list_folder(request)
-    if list_folder_ret[u'code'] == 0:
-        print(True) 
-    else:
-        print("%s, appid/secret_id/secret_key/region/bucket invalid"% list_folder_ret[u'message'])
-
-def cmd_mkdir(COSDIR):
-    cosdir = COSDIR.decode('utf-8')
-    request = CreateFolderRequest(bucket, cosdir)
-    create_folder_ret = cos_client.create_folder(request)
-    if create_folder_ret[u'code'] == 0:
-        print("mkdir cos://%s%s OK" % (bucket,COSDIR))
-    else:
-        print(create_folder_ret[u'message'])
-
-def cmd_put(LOCALFILE,COSFILE):
-    localfile = LOCALFILE.decode('utf-8')
-    cosfile = COSFILE.decode('utf-8')
-    request = UploadFileRequest(bucket, cosfile, localfile)
-    request.set_insert_only(0)
-    upload_file_ret = cos_client.upload_file(request)
-    if upload_file_ret[u'code'] == 0:
-        print("put cos://%s%s OK" % (bucket,COSFILE))
-    else:
-        print(upload_file_ret[u'message'])
-
-def loginit():
-    global config
-    if (log_file_name == ""):
-        return
-    log_level = logging.ERROR
-    if log_level == 0:
-        log_level = logging.DEBUG
-    if log_level == 1:
-        log_level = logging.INFO
-    if log_level == 2:
-        log_level = logging.WARNING
-
-    #定义一个RotatingFileHandler，最多备份5个日志文件，每个日志文件最大20M
-    logger = logging.getLogger("")
-    Rthandler = RotatingFileHandler(log_file_name, maxBytes= 20*1024*1024,backupCount=5)
-    Rthandler.setLevel(log_level)
-    formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s')
-    Rthandler.setFormatter(formatter)
-    logger.addHandler(Rthandler)
-    #输出日志到屏幕
-    console = logging.StreamHandler()
-    console.setFormatter(formatter)
-    if (log_out_to_screen == 1):
-        logger.addHandler(console)
-
-    logger.setLevel(log_level)
-    return logger
-
-#日期相关操作
-class Dateop():
-    @staticmethod
-    def isValidDate(str):
-        try:
-            time.strptime(str, "%Y""%m""%d")
-            return True
-        except:
-            return False
-
-    @staticmethod
-    def getdaystr(n=0):
-        dt = date.today()-timedelta(days=n)
-        tt = dt.timetuple()
-        daystr = strftime("%Y""%m""%d",tt)
-        return daystr
-
-    @staticmethod
-    def cmpDateAgo(t1,t2):
-        if (Dateop.isValidDate(t1)==False or Dateop.isValidDate(t2)==False):
-            return False
-        if (int(t1) <= int (t2)):
-            return True
-        return False
-
-    @staticmethod
-    def isNeedDeleteDir(dirname, n=0):
-        if (len(dirname) != 8):
-            return False
-        if Dateop.isValidDate(dirname) == False:
-            return False
-        d2 = Dateop.getdaystr(n);
-        if Dateop.cmpDateAgo(dirname, d2):
-            return True
-        return False
-#删除文件统计
-class FileStat():
-    global cos_log
-    def __init__(self):
-        self.delfilesuccnum = 0
-        self.deldirsuccnum = 0
-        self.delfilefailnum = 0
-        self.deldirfailnum = 0
-        self.lock = threading.Lock()
- 
-    def addDelFileFailNum(self,num=1):
-        self.lock.acquire(1)
-        self.delfilefailnum += num
-        self.lock.release()
-    def addDelDirFailNum(self,num=1):
-        self.lock.acquire(1)
-        self.deldirfailnum += num
-        self.lock.release()
-    def addDelDirSuccNum(self, num=1):
-        self.lock.acquire(1)
-        self.deldirsuccnum += num
-        self.lock.release()
-    def addDelFileSuccNum(self, num=1):
-        self.lock.acquire(1)
-        self.delfilesuccnum += num
-        self.lock.release()
-    def printStat(self):
-        msg ="".join(["delfilesuccnum=",str(self.delfilesuccnum),
-                ",delfilefailnum=",str(self.delfilefailnum),
-                ",deldirsuccnum=",str(self.deldirsuccnum),
-                ",deldirfailnum=",str(self.deldirfailnum)])
-        print(msg) 
-    def logStat(self):
-        curtime = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
-        log = ''.join(["delfilenum=",str(self.delfilesuccnum),
-            ",deldirnum=",str(self.deldirsuccnum),",delfilefailnum=",
-            str(self.delfilefailnum),",deldirfailnum=",str(self.deldirfailnum)])
-        cos_log.info(log)
-
-#执行时间统计
-class TimeStat(object):
-    global cos_log
-    def __init__(self):
-	self.start()
-    def start(self):
-        self.start = datetime.datetime.now()
-        self.t1 = time.time()
-        msg = "delete task started  ..........."
-        cos_log.info(msg)
-    def end(self):
-        self.end = datetime.datetime.now()
-        self.t2 = time.time()
-        msg = "delete task ended\n\nrm task finished,\ntimecost:"+str(self.t2-self.t1) + " (s)"
-        cos_log.info(msg)
-
-#删除文件列表中的文件
-def delfiles(cos_client, bucket, filelist):
-    for f in filelist:
-        delfile(cos_client, bucket, f)
-
-def delfolders(cos_client, bucket, folderlist):
-    for f in folderlist:
-        delfolder(cos_client, bucket, f)
-
-#文件夹删除
-def delfolder(cos_client, bucket, folder):
-    global stat
-    global cos_log
-    if not folder:
-        return 0
-    delfolderreq = DelFolderRequest(bucket, folder)
-    retry = 0
-    while (retry < MAX_RETRY_TIMES):
-        ret = cos_client.del_folder(delfolderreq)
-        #msg = "delfolder fail, bucket="+bucket+",folder="+folder+ret['message']
-        msg = "delfolder fail, bucket="+bucket+",folder="+folder+str(ret.get('message'))
-        if (ret['code'] == 0):
-            break
-        elif (ret['code'] == -166):
-            cos_log.warning(msg)
-            break
-        #操作太频繁,频控
-        elif (ret['code'] == -71):
-            sleep(random.randint(1,5))
-            cos_log.warning(msg)
-            retry += 1
-            continue
-        #文件夹非空
-        elif (ret['code'] == -173):
-            break
-        else:
-            cos_log.warning(msg)
-            retry += 1
-    if (ret['code'] != 0 and  ret['code'] != -166):
-        stat.addDelDirFailNum()
-        #cos_log.error("delfolder fail, bucket="+bucket+",folder="+folder+ret['message'])
-        cos_log.error("delfolder fail, bucket="+bucket+",folder="+folder+str(ret.get('message')))
-        return ret['code']
-    if (ret['code'] == 0):
-        stat.addDelDirSuccNum()
-        msg = "delfolder success, bucket="+bucket+",folder="+folder
-        cos_log.info(msg)
-    return 0
-
-#文件删除
-def delfile(cos_client, bucket, filepath):
-    global stat
-    global cos_log
-    delfilereq = DelFileRequest(bucket, filepath)
-    retry = 0
-    while (retry < MAX_RETRY_TIMES):
-        ret = cos_client.del_file(delfilereq)
-        #msg = "delfile fail bucket="+bucket+",file="+filepath+ret['message']
-        msg = "delfile fail bucket="+bucket+",file="+filepath+str(ret.get('message'))
-        if (ret['code'] == 0):
-            break
-        #文件不存在
-        elif (ret['code'] == -166):
-            cos_log.warning(msg)
-            break
-        #单目录写操作过快
-        elif (ret['code'] == -143):
-            sleep(random.randint(1,5))
-            cos_log.warning(msg)
-            retry += 1
-            continue
-        #操作太频繁,频控
-        elif (ret['code'] == -71):
-            sleep(random.randint(1,5))
-            cos_log.warning(msg)
-            retry += 1
-            continue
-        else:
-            cos_log.warning(msg)
-            retry += 1
-            continue
-    if (ret['code'] != 0 and  ret['code'] != -166):
-        stat.addDelFileFailNum()
-        #cos_log.error("delfile fail, bucket="+bucket+",file="+filepath+ret['message'])
-        cos_log.error("delfile fail, bucket="+bucket+",file="+filepath+str(ret.get('message')))
-        return ret['code']
-    if (ret['code'] == 0):
-        stat.addDelFileSuccNum()
-        msg = "delfile success, bucket="+bucket+",file="+filepath
-        cos_log.info(msg)
-    return 0
-
-#递归文件夹进行文件删除
-def delete_r(cos_client, bucket, path, thread_pool_file):
-    global stat
-    global config
-    global cos_log
-    cos_log.debug("delete_r bucket:"+bucket+",path:"+path)
-    context = u""
-    #递归文件夹
-    while True:
-        listfolderreq = ListFolderRequest(bucket, path, 1000, u'', context)
-        retry = 0
-        while (retry < MAX_RETRY_TIMES):
-            listret = cos_client.list_folder(listfolderreq)
-            if listret['code'] != 0 :
-                retry += 1
-                sleep(random.randint(1,3))
-                continue
-            else:
-                break
-        if (listret['code'] != 0):
-            #cos_log.error("delete_r: list folder fail:"+path +",return msg:"+ listret['message'])
-            cos_log.error("delete_r: list folder fail:"+path +",return msg:"+ str(listret.get('message')))
-            return listret['code']
-        if (len(listret['data']['infos']) == 0):
-            break;
-        filelist = []
-        dirlist = []
-        for info in listret['data']['infos']:
-            fullname = path + info['name']
-            #list出来的文件列表中文件夹和文件本身是混杂一起的
-            if info.has_key('filesize'):
-                filelist.append(fullname)
-                if (len(filelist) >= ONE_TASK_DEL_FILE_NUMS):
-                    args = [cos_client, bucket, filelist]
-                    args_tuple = (args,None)
-                    args_list = [args_tuple]
-                    requests = threadpool.makeRequests(delfiles, args_list)
-                    for req in requests:
-                        thread_pool_file.putRequest(req)
-                        filelist = []
-                        continue
-                else:
-                    pass
-            else:
-                dirlist.append(fullname)
-                if (len(dirlist) >= ONE_TASK_DEL_FILE_NUMS):
-                    args = [cos_client, bucket, dirlist]
-                    args_tuple = (args,None)
-                    args_list = [args_tuple]
-                    requests = threadpool.makeRequests(delfolders, args_list)
-                    for req in requests:
-                        thread_pool_file.putRequest(req)
-                        dirlist = []
-                        continue
-                else:
-                    pass
-                pass
-
-        if (len(filelist) > 0):
-            args = [cos_client, bucket, filelist]
-            args_tuple = (args,None)
-            args_list = [args_tuple]
-            requests = threadpool.makeRequests(delfiles, args_list)
-            for req in requests:
-                thread_pool_file.putRequest(req)
-                filelist = []
-        else:
-            pass
-		
-        if (len(dirlist) > 0):
-            args = [cos_client, bucket, dirlist]
-            args_tuple = (args,None)
-            args_list = [args_tuple]
-            requests = threadpool.makeRequests(delfolders, args_list)
-            for req in requests:
-                thread_pool_file.putRequest(req)
-                filelist = []
-        else:
-            pass
- 
-        cos_log.debug("delete_r thread pool file waiting\n")
-        thread_pool_file.wait()
-        cos_log.debug("delete_r thread pool file waiting end\n")
-
-        if (listret['data']['listover'] == False):
-            context = listret['data']['context']
-            continue
-        else:
-            break
-
-    stat.logStat()
-    return 0
-
-#支持Ctrl+C终止程序
-class Watcher():
-    def __init__(self):
-        self.child = os.fork()
-        if self.child == 0:
-            return
-        else:
-            self.watch()
-             
-    def watch(self):
-        global cos_log
-        try: 
-            os.wait()
-        except KeyboardInterrupt:
-            cos_log.ERROR("ctrl+c terminated coscmd, exiting...")                                                                                        
-            self.kill()
-        sys.exit()
-    def kill(self):
-        try: 
-            os.kill(self.child, signal.SIGKILL)
-        except OSError:
-            pass
-
-def cmd_rm(COSDIR):
-    global thread_pool
-    global cos_log
-    global stat 
-    cos_log = loginit()
-    stat = FileStat()
-    timestat = TimeStat()
-    if HAS_FORK:
-      Watcher()
-    path = COSDIR.decode('utf-8')
-    thread_pool_dir = threadpool.ThreadPool(dir_thread_num)
-    thread_pool_file = threadpool.ThreadPool(file_thread_num)
-    cos_log.debug("bucket:"+bucket +",path:"+path)
-    args = [cos_client, bucket, path, thread_pool_file]
-    args_tuple = (args, None)
-    args_list = [args_tuple]
-    requests = threadpool.makeRequests(delete_r, args_list)
-    for req in requests:
-        thread_pool_dir.putRequest(req)
-
-    cos_log.debug("thread_pool_dir waiting.....\n")
-    thread_pool_dir.wait()
-    thread_pool_dir.dismissWorkers(dir_thread_num, True)
-    cos_log.debug("thread_pool_dir wait end.....\n")
-
-    timestat.end()
-    stat.logStat()
-
-if sys.argv[1] in ['config','ls','mkdir','put','rm','delete','del'] and len(sys.argv) >= 3:
-    if sys.argv[1] == 'config':
-        parser = OptionParser()
-        parser.add_option("-a", "--appid", dest="appid", help="specify appid")
-        parser.add_option("-i", "--id", dest="secret_id", help="specify secret id")
-        parser.add_option("-k", "--key", dest="secret_key", help="specify secret key")
-        parser.add_option("-r", "--region", dest="region", help="specify region")
-        parser.add_option("-b", "--bucket", dest="bucket", help="specify bucket")
-        (options, args) = parser.parse_args()
-        CMD_LIST['config'] = cmd_configure    
-        CMD_LIST['config'](args, options)
-    if sys.argv[1] == 'ls':
-        cmd_loadconfigure()
-        cos_client = CosClient(appid, secret_id, secret_key, region)
-        COSDIR = sys.argv[2]
-        cmd_lsdir(COSDIR)
-    if sys.argv[1] == 'mkdir':
-        cmd_loadconfigure()
-        cos_client = CosClient(appid, secret_id, secret_key, region)
-        COSDIR = sys.argv[2]
-        cmd_mkdir(COSDIR)
-    if sys.argv[1] == 'put' and len(sys.argv) == 4:
-        cmd_loadconfigure()
-        cos_client = CosClient(appid, secret_id, secret_key, region)
-        LOCALFILE = sys.argv[2]
-        COSFILE = sys.argv[3]
-        cmd_put(LOCALFILE,COSFILE)
-    if sys.argv[1] in ('rm','delete','del'):
-        cmd_loadconfigure()
-        cos_client = CosClient(appid, secret_id, secret_key, region)
-        COSDIR = sys.argv[2]
-        path = COSDIR.decode('utf-8') 
-        cmd_rm(path)
-else:
-    print(HELP)
-    exit()

versions.txt

@@ -92,4 +92,4 @@ bison_version=2.7.1
 python_version=2.7.13
 setuptools_version=32.0.0
 pip_version=9.0.1
-fail2ban_version=0.9.7
+fail2ban_version=0.10