FastDFS加Redis實作自訂檔名儲存海量文件-Python教學-PHP中文網

FastDFS加Redis實作自訂檔名儲存海量文件

高洛峰

發布： 2016-10-18 10:27:51

原創

1915 人瀏覽過

FastDFS非常適合儲存大量的小文件，遺憾的是本身不支援自訂文件名，文件名是儲存成功以後根據儲存位置產生的一個file_id。許多應用程式場景都必須使用自訂檔案名，在不修改其原始碼的情況下，可以在儲存客戶端fdfs_client增加一個用來儲存自訂檔案名稱和fastdfs的file_id之間的映射關係的資料庫間接實作自訂文件名的訪問和訪問，在這裡我們選用了reids。順便說一下，淘寶也有一個類似於FastDFS的檔案儲存系統TFS，對於自訂檔案名，它是用mysql來儲存映射關係的，我認為在高並發存取下mysql本身就是瓶頸，因此在這個方案中採用了redis。

準備工作：

fastdfs環境安裝...略...（官方：https://code.google.com/p/fastdfs/）

redis環境安裝...略...(官方： http://redis.io/)

用python實現，因此需要安裝fastdfs的python客戶端(下載：https://fastdfs.googlecode.com/files/fdfs_client-py-1.2.6.tar.gz)

python的redis客戶端，到https://pypi.python.org/pypi/redis下載

# -*- coding: utf-8 -*-
import setting
from fdfs_client.client import *
from fdfs_client.exceptions import *
  
from fdfs_client.connection import *
  
import redis
import time
import logging
import random
  
logging.basicConfig(format=&#39;[%(levelname)s]: %(message)s&#39;, level=logging.DEBUG)
logger = logging.getLogger(__name__)
logger.setLevel(logging.DEBUG)
  
  
class RedisError(Exception):
     def __init__(self, value):
         self.value = value
     def __str__(self):
         return repr(self.value)
  
class fastdfsClient(Fdfs_client):
    def __init__(self):
        self.tracker_pool = ConnectionPool(**setting.fdfs_tracker)
        self.timeout  = setting.fdfs_tracker[&#39;timeout&#39;]
        return None
  
    def __del__(self):
        try:
            self.pool.destroy()
            self.pool = None
        except:
            pass
  
class fastdfs(object):
    def __init__(self):
        &#39;&#39;&#39;
        conf_file:配置文件
        &#39;&#39;&#39;
        self.fdfs_client = fastdfsClient()
        self.fdfs_redis = []
        for i in setting.fdfs_redis_dbs:
            self.fdfs_redis.append(redis.Redis(host=i[0], port=i[1], db=i[2]))
  
    def store_by_buffer(self,buf,filename=None,file_ext_name = None):
        &#39;&#39;&#39;
        buffer存储文件
        参数：
        filename:自定义文件名，如果不指定，将远程file_id作为文件名
        file_ext_name:文件扩展名（可选），如果不指定，将根据自定义文件名智能判断
        返回值：
        {
        &#39;group&#39;:组名,
        &#39;file_id&#39;:不含组名的文件ID,
        &#39;size&#39;:文件尺寸,
        &#39;upload_time&#39;:上传时间
        }
        &#39;&#39;&#39;
        if filename and  random.choice(self.fdfs_redis).exists(filename):
            logger.info(&#39;File(%s) exists.&#39;%filename)
            return   random.choice(self.fdfs_redis).hgetall(filename)
        t1 = time.time()
#        try:
        ret_dict = self.fdfs_client.upload_by_buffer(buf,file_ext_name)
#        except Exception,e:
#            logger.error(&#39;Error occurred while uploading: %s&#39;%e.message)
#            return None
        t2 = time.time()
        logger.info(&#39;Upload file(%s) by buffer, time consume: %fs&#39; % (filename,(t2 - t1)))
        for key in ret_dict:
            logger.debug(&#39;[+] %s : %s&#39; % (key, ret_dict[key]))
        stored_filename = ret_dict[&#39;Remote file_id&#39;]
        stored_filename_without_group = stored_filename[stored_filename.index(&#39;/&#39;)+1:]
        if not filename:
            filename =stored_filename_without_group
        vmp = {&#39;group&#39;:ret_dict[&#39;Group name&#39;],&#39;file_id&#39;:stored_filename_without_group,&#39;size&#39;:ret_dict[&#39;Uploaded size&#39;],&#39;upload_time&#39;:int(time.time()*1000)}
        try:
            for i in self.fdfs_redis:
                if not i.hmset(filename,vmp):
                    raise RedisError(&#39;Save Failure&#39;)
                logger.info(&#39;Store file(%s) by buffer successful&#39; % filename)
        except Exception,e:
            logger.error(&#39;Save info to Redis failure. rollback...&#39;)
            try:
                ret_dict = self.fdfs_client.delete_file(stored_filename)
            except Exception,e:
                logger.error(&#39;Error occurred while deleting: %s&#39;%e.message)
            return None
        return vmp
  
    def remove(self,filename):
        &#39;&#39;&#39;
        删除文件,
        filename是用户自定义文件名
        return True|False
        &#39;&#39;&#39;
        fileinfo = random.choice(self.fdfs_redis).hgetall(filename)
        stored_filename = &#39;%s/%s&#39;%(fileinfo[&#39;group&#39;],fileinfo[&#39;file_id&#39;])
        try:
            ret_dict = self.fdfs_client.delete_file(stored_filename)
            logger.info(&#39;Remove stored file successful&#39;)
        except Exception,e:
            logger.error(&#39;Error occurred while deleting: %s&#39;%e.message)
            return False
        for i in self.fdfs_redis:
            if not i.delete(filename):
                logger.error(&#39;Remove fileinfo in redis failure&#39;)
        logger.info(&#39;%s removed.&#39;%filename)
        return True
  
    def download(self,filename):
        &#39;&#39;&#39;
        下载文件
        返回二进制
        &#39;&#39;&#39;
        finfo = self.getInfo(filename)
        if finfo:
            ret = self.fdfs_client.download_to_buffer(&#39;%s/%s&#39;%(finfo[&#39;group&#39;],finfo[&#39;file_id&#39;]))
            return ret[&#39;Content&#39;]
        else:
            logger.debug(&#39;%s is not exists&#39;%filename)
            return None
  
    def list(self,pattern=&#39;*&#39;):
        &#39;&#39;&#39;
        列出文件列表
        &#39;&#39;&#39;
        return random.choice(self.fdfs_redis).keys(pattern)
  
    def getInfo(self,filename):
        &#39;&#39;&#39;
        获得文件信息
        return:{
        &#39;group&#39;:组名,
        &#39;file_id&#39;:不含组名的文件ID,
        &#39;size&#39;:文件尺寸,
        &#39;upload_time&#39;:上传时间
        }
        &#39;&#39;&#39;
        return random.choice(self.fdfs_redis).hgetall(filename)

登入後複製

設定：

# -*- coding: utf-8 -*-
#fastdfs tracker, multiple tracker supported
fdfs_tracker = {
&#39;host_tuple&#39;:(&#39;192.168.2.233&#39;,&#39;192.168.2.234&#39;),
&#39;port&#39;:22122,
&#39;timeout&#39;:30,
&#39;name&#39;:&#39;Tracker Pool&#39;
}
#fastdfs meta db, multiple redisdb supported
fdfs_redis_dbs = (
    (&#39;192.168.2.233&#39;,6379,0),
    (&#39;192.168.2.233&#39;,6379,1)
)

登入後複製