FastDFS分布式文件存储

FastDFS文件上传

简介：

主要解决：大容量的文件存储和高并发访问的问题

论坛：https://bbs.chinaunix.net

下载网站：https://sourceforge.net/projects/fastdfs/files/

安装参考：https://www.cnblogs.com/cxygg/p/16000425.html

启动方式：参考上的方式启动不了，使用如下方式启动：

service fdfs_trackerd start

service fdfs_trackerd status

service fdfs_trackerd stop

service fdfs_storaged start

service fdfs_storaged status

service fdfs_storaged stop

上传文件命令：

fdfs_upload_file /etc/fdfs/client.conf /home/b.txt

java操作使用：

<dependency><groupId>cn.bestwu</groupId><artifactId>fastdfs-client-java</artifactId><version>1.27</version>
</dependency>
<dependency><groupId>org.apache.commons</groupId><artifactId>commons-lang3</artifactId><version>3.4</version>

fastdfs.connect_timeout_in_seconds=10
fastdfs.network_timeout_in_seconds=30
fastdfs.charset=UTF-8
fastdfs.tracker_servers=192.168.0.138:22122
#/etc/fdfs/tracker.conf中http.server_port配置的完全一致
fastdfs.http_tracker_http_port=8080

package org.example;import org.csource.common.MyException;
import org.csource.common.NameValuePair;
import org.csource.fastdfs.*;import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.Properties;/*** Hello world!*/
public class App {private static String dirPath = "C:\\Users\\Administrator\\Desktop\\notes\\";public static void main(String[] args) throws IOException, MyException {//加载配置信息Properties properties = new Properties();properties.load(App.class.getClassLoader().getResourceAsStream("fdfs.properties"));ClientGlobal.initByProperties(properties);//初始化客户端对象，StorageClient,存储客户端TrackerClient trackerClient = new TrackerClient();TrackerServer trackerServer = trackerClient.getConnection();StorageServer storageServer = trackerClient.getStoreStorage(trackerServer);StorageClient storageClient = new StorageClient(trackerServer, storageServer);//文件上传//读取文件字节内容到一个字节数组中，文件必须完整的存在一个数组中。FileInputStream in = new FileInputStream(dirPath + "Redis.md");//根据文件的字节长度，创建一个等长的数组byte[] data = new byte[in.available()];//把文件内容完整的读取到数组中in.read(data, 0, data.length);//元数据，由使用者自定义，是名值对形式NameValuePair[] nvps = new NameValuePair[]{new NameValuePair("name", "Redis.md")};//上传文件，返回一个字符串数组，长度为2。0下标代表组名。1下标代表文件路径及文件名// 如： [0]=group1;  [1]=M00/00/00/dadfsfsdfs.mdString[] result = storageClient.upload_file(data, "md", nvps);System.out.println("返回结果数组长度:" + result.length);System.out.println("rest[0]=" + result[0]);System.out.println("rest[1]=" + result[1]);}
}

//工具

package org.example.utils;import org.csource.common.MyException;
import org.csource.common.NameValuePair;
import org.csource.fastdfs.*;
import org.example.App;import java.io.FileInputStream;
import java.io.IOException;
import java.util.Properties;public class FastDFSUtils {private FastDFSUtils() {}private static final Properties properties = new Properties();private static StorageClient storageClient;static {try {//加载配置信息properties.load(App.class.getClassLoader().getResourceAsStream("fdfs.properties"));ClientGlobal.initByProperties(properties);//初始化客户端对象，StorageClient,存储客户端TrackerClient trackerClient = new TrackerClient();TrackerServer trackerServer = trackerClient.getConnection();StorageServer storageServer = trackerClient.getStoreStorage(trackerServer);storageClient = new StorageClient(trackerServer, storageServer);} catch (Exception e) {e.printStackTrace();throw new ExceptionInInitializerError(e);}}//上传public static String uploadFile(byte[] datas, String extName, NameValuePair[] nvps) {try {String[] resutl;resutl = storageClient.upload_file(datas, extName, nvps);String path = resutl[0] + "/" + resutl[1];return path;} catch (Exception e) {e.printStackTrace();//上传失败return null;}}//下载public static byte[] download(String group,String path){try {byte[] bytes = storageClient.download_file(group, path);return bytes;} catch (Exception e) {e.printStackTrace();return null;}}public static  NameValuePair[] getMetaDatas(String group,String path){try {NameValuePair[] metadata = storageClient.get_metadata(group, path);return metadata;} catch (Exception e) {e.printStackTrace();return null;}}public static boolean delete(String group,String path) throws Exception{try {int status = storageClient.delete_file(group,path);return status == 0;} catch (Exception e) {e.printStackTrace();return false;}}}

package org.example;import org.csource.common.MyException;
import org.csource.common.NameValuePair;
import org.csource.fastdfs.*;
import org.example.utils.FastDFSUtils;import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.Properties;
import java.util.UUID;/*** Hello world!*/
public class App {private static String dirPath = "C:\\Users\\Administrator\\Desktop\\notes\\";public  static  void uploadFile() throws Exception{FileInputStream in = new FileInputStream(dirPath + "Redis.md");int length = in.available();byte[] datas = new byte[length];in.read(datas,0,length);NameValuePair[] nvps = new NameValuePair[]{new NameValuePair("name","Redis.md")
//                new NameValuePair("length",length + "")};String result = FastDFSUtils.uploadFile(datas,"md",nvps);System.out.println(result);}public static void download() throws Exception{String group = "group1";String path = "M00/00/00/wKgAimRDupOAC57HAAAtdrasJ7c7839.md";NameValuePair[]  nvps = FastDFSUtils.getMetaDatas(group,path);byte[] data = FastDFSUtils.download(group, path);String loclaFileName = "";for (NameValuePair nvp : nvps) {if (nvp.getName().equals("name")){loclaFileName = nvp.getValue();}}if ("".equals(loclaFileName)){//元数据中没有文件名，设置文件名为随机命名loclaFileName = UUID.randomUUID().toString() + path.substring(path.lastIndexOf("."+1));}FileOutputStream out = new FileOutputStream("F:\\"+loclaFileName);out.write(data,0,data.length);out.flush();out.close();}public static void delete() throws Exception {String group = "group1";String path = "M00/00/00/wKgAimRDupOAC57HAAAtdrasJ7c7839.md";boolean delete = FastDFSUtils.delete(group, path);System.out.println(delete? "删除成功":"删除失败");}public static void main(String[] args) throws Exception {//        uploadFile();
//        download();delete();//        //加载配置信息
//        Properties properties = new Properties();
//        properties.load(App.class.getClassLoader().getResourceAsStream("fdfs.properties"));
//        ClientGlobal.initByProperties(properties);
//
//        //初始化客户端对象，StorageClient,存储客户端
//        TrackerClient trackerClient = new TrackerClient();
//        TrackerServer trackerServer = trackerClient.getConnection();
//        StorageServer storageServer = trackerClient.getStoreStorage(trackerServer);
//        StorageClient storageClient = new StorageClient(trackerServer, storageServer);
//
//        //文件上传
//        //读取文件字节内容到一个字节数组中，文件必须完整的存在一个数组中。
//        FileInputStream in = new FileInputStream(dirPath + "Redis.md");
//        //根据文件的字节长度，创建一个等长的数组
//        byte[] data = new byte[in.available()];
//        //把文件内容完整的读取到数组中
//        in.read(data, 0, data.length);
//        //元数据，由使用者自定义，是名值对形式
//        NameValuePair[] nvps = new NameValuePair[]{
//                new NameValuePair("name", "Redis.md")
//        };
//        //上传文件，返回一个字符串数组，长度为2。0下标代表组名。1下标代表文件路径及文件名
//        // 如： [0]=group1;  [1]=M00/00/00/dadfsfsdfs.md
//        String[] result = storageClient.upload_file(data, "md", nvps);
//        System.out.println("返回结果数组长度:" + result.length);
//        System.out.println("rest[0]=" + result[0]);
//        System.out.println("rest[1]=" + result[1]);}
}

安装nginx的依赖

yum install -y gcc gcc-c++ make automake autoconf libtool pcre pcre-devel zlib zlib-devel openssl openssl-devel

上传nginx nginx-moduel

解压 nginx-moduel

vim config

看文件中的一个目录 一般删除local

ls /usr/local/include/fastdfs

ls /usr/local/include/fastcommon

ls /usr/local/lib

/etc/fdfs/mod_fastdfs.conf 文件就在当前的src 目录下 进行 cp mod_fastdfs.conf /etc/fdfs/

解压 nginx

创建文件夹

mkdir -p  /usr/local/nginx
mkdir -p  /var/run/nginx
mkdir -p  /var/lock
mkdir -p  /var/log/nginx
mkdir -p  /var/temp/nginx

启动nginx

./configure \
--prefix=/usr/local/nginx \
--pid-path=/var/run/nginx/nginx.pid \
--lock-path=/var/lock/nginx.lock \
--error-log-path=/var/log/nginx/error.log \
--http-log-path=/var/log/nginx/access.log \
--with-http_gzip_static_module \
--http-client-body-temp-path=/var/temp/nginx/client \
--http-proxy-temp-path=/var/temp/nginx/proxy \
--http-fastcgi-temp-path=/var/temp/nginx/fastcgi \
--http-uwsgi-temp-path=/var/temp/nginx/uwsgi \
--http-scgi-temp-path=/var/temp/nginx/scgi \
--add-module=/opt/tools/fastdfs-nginx-module/src

修改

vim /etc/fdfs/mod_fastdfs.conf

tracker_server=192.168.0.138:22122

url_have_group_name = true

编译

有Makefile文件 的直接执行

make

make install

cd /usr/local/nginx

vim nginx.conf

user root;

/opt/tools/FastDFS/conf

cp http.conf /etc/fdfs/

cp mime.types /etc/fdfs/

sbin/nginx

vim nginx.conf

server{
listen 8888;
server_name localhost;
location ~/group([1-9])/M00 {
ngx_fastdfs_module;

}

}

sbin/nginx -s quit

sbin/nginx

http://192.168.0.138:8888/group1/M00/00/00/wKgAimRFUpeAbRwAAAFNn0YHOlA48.jpeg