报表下载工具

1.需求说明

我有一堆文件的Url地址， 现在需要按照企业，项目和报表类型分类下载到对应的文件夹中

2.相关实体类

• 企业文件夹定义

package com.vz.utils.report;import lombok.Data;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;/*** @author visy.wang* @description: 企业文件夹* @date 2023/7/19 16:49*/
@Data
public class EnterpriseDto {/*** 企业名称（文件夹名）*/private String name;/*** 项目文件夹列表*/private List<ProjectDto> projectList;/*** 添加项目*/public void addProject(ProjectDto project){if(Objects.isNull(projectList)){projectList = new ArrayList<>();}projectList.add(project);}
}

• 项目文件夹定义

package com.vz.utils.report;import lombok.Data;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;/*** @author visy.wang* @description: 项目文件夹* @date 2023/7/19 16:50*/
@Data
public class ProjectDto {/*** 项目名称（文件夹名）*/private String name;/*** 报表文件夹列表*/private List<ReportDto> reportList;/*** 添加报表快捷方式* @param reportType 数据类型* @param urlList 下载Url列表*/public void addReport(ReportTypeEnum reportType, List<String> urlList){if(Objects.isNull(reportList)){reportList = new ArrayList<>();}ReportDto reportDto = new ReportDto();reportDto.setReportType(reportType);reportDto.setUrlList(urlList);reportList.add(reportDto);}
}

• 报表文件夹定义

package com.vz.utils.report;import lombok.Data;
import java.util.List;/*** @author visy.wang* @description: 报表文件夹* @date 2023/7/19 17:23*/
@Data
public class ReportDto {/*** 报表类型（文件夹名）*/private ReportTypeEnum reportType;/*** 文件下载地址列表*/private List<String> urlList;
}

• 报表类型枚举

package com.vz.utils.report;import lombok.AllArgsConstructor;
import lombok.Getter;/*** @author visy.wang* @description: 报表类型枚举* @date 2023/7/19 16:57*/
@Getter
@AllArgsConstructor
public enum ReportTypeEnum {SETTLEMENT("结算单", "结算单"),BI("BI", "BI"),DATA_REPORT("数据报表", "数据报表"),CONTRACT("合同", "合同");//文件夹名private final String folder;//类型描述private final String desc;
}

3.核心下载类

package com.vz.utils.report;import cn.hutool.core.io.FileUtil;
import cn.hutool.core.net.URLDecoder;
import cn.hutool.http.HttpResponse;
import cn.hutool.http.HttpUtil;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
import com.server.utils.mail.MailUtil;
import lombok.extern.slf4j.Slf4j;
import org.springframework.http.HttpStatus;
import org.springframework.util.CollectionUtils;
import org.springframework.util.StringUtils;import java.io.File;
import java.nio.charset.StandardCharsets;
import java.util.*;
import java.util.concurrent.*;/*** @author visy.wang* @description: 下载工具* @date 2023/7/19 16:47*/
@Slf4j
public class DownUtil {private static final Map<String,Boolean> folderCreateFlag = new HashMap<>();private static final String rootPath = "/usr/temp_dir/data_report"; //Linux文件存放根路径//private static final String rootPath = "E:\\test\\down"; //测试用，Windows系统// 创建线程池private static final ExecutorService pool;static {ThreadFactory namedThreadFactory = new ThreadFactoryBuilder().setNameFormat("report-down-pool-%d").build();pool = new ThreadPoolExecutor(20, 100, 5000L, TimeUnit.MILLISECONDS,new LinkedBlockingQueue<>(5000), namedThreadFactory, new ThreadPoolExecutor.AbortPolicy());}/*** 下载* @param enterpriseList 企业列表* @param name 下载类型 (根文件夹名)*/public static void download(List<EnterpriseDto> enterpriseList, String name){//创建基础文件夹String baseFolder = rootPath + File.separator + name;createFolder(baseFolder);if(CollectionUtils.isEmpty(enterpriseList)){log.info("企业列表为空，下载结束！！！");return;}int downSuccessCount = 0, downFailureCount = 0, downRepeatCount = 0;long startTime = System.currentTimeMillis();List<CompletableFuture<Boolean>> futureList = new ArrayList<>();//遍历企业并创建文件夹for(EnterpriseDto enterprise: enterpriseList) {//创建企业文件夹String enterpriseFolder = baseFolder + File.separator + enterprise.getName();createFolder(enterpriseFolder);//处理项目列表List<ProjectDto> projectList = enterprise.getProjectList();if(CollectionUtils.isEmpty(projectList)){log.info("项目列表为空，文件夹：{}", enterpriseFolder);continue;}//遍历项目并创建文件夹for (ProjectDto project : projectList) {//创建项目文件夹String projectFolder = enterpriseFolder + File.separator + project.getName();createFolder(projectFolder);//处理数据列表List<ReportDto> reportList = project.getReportList();if(CollectionUtils.isEmpty(reportList)){log.info("数据列表为空，文件夹：{}", projectFolder);continue;}//遍历数据并创建文件夹for (ReportDto report : reportList) {ReportTypeEnum reportType = report.getReportType();//创建报表文件夹String reportFolder = projectFolder + File.separator + reportType.getFolder();createFolder(reportFolder);List<String> urlList = report.getUrlList();if(CollectionUtils.isEmpty(urlList)){log.info("文件URL列表为空，文件夹：{}", reportFolder);continue;}for (String url : urlList) {//下载文件到当前文件夹if(!StringUtils.hasText(url)){log.info("{}下载失败，URL为空", reportType.getDesc());downFailureCount ++;continue;}log.info("正在下载{}，URL：{}", reportType.getDesc(), url);try{String uri = URLDecoder.decode(url.trim(), StandardCharsets.UTF_8);//从Url提取文件名int index = uri.lastIndexOf("/");String fileName = index==-1 ? uri : uri.substring(index+1);//检查是否下载过String filePath = reportFolder + File.separator + fileName;if(new File(filePath).exists()){log.info("已下载过，无需再次下载，URL:{}", url);downRepeatCount ++;continue;}//下载并保存CompletableFuture<Boolean> future = CompletableFuture.supplyAsync(() -> {return downFile(reportType, url, filePath);}, pool);futureList.add(future);}catch (Exception e){downFailureCount ++;log.info("{}下载失败，error:{}，URL：{}", reportType.getDesc(), url, e.getMessage(), e);}}}}}for (CompletableFuture<Boolean> future: futureList) {try {if(future.get()){downSuccessCount ++;}else{downFailureCount ++;}}catch (Exception e){downFailureCount ++;e.printStackTrace();}}long spendTime = System.currentTimeMillis() - startTime;log.info("【{}】全部下载完成，下载成功数：{}，下载失败数：{}，下载重复数：{}，总耗时：{}ms",name, downSuccessCount, downFailureCount, downRepeatCount, spendTime);}/*** 下载文件并保存在服务器* @param reportType 报表类型* @param url 下载地址* @param filePath 本地保存路径（全路径，含文件名和后缀）* @return 是否成功*/private static boolean downFile(ReportTypeEnum reportType, String url, String filePath){try(HttpResponse response = HttpUtil.createGet(url).execute()){if(response.getStatus() == HttpStatus.OK.value()){//写入本地文件FileUtil.writeBytes(response.bodyBytes(), filePath);return true;}else{throw new RuntimeException("HttpStatus="+response.getStatus());}}catch (Exception e){log.info("{}下载失败，Error：{}，URL：{}", reportType.getDesc(), e.getMessage(), url, e);return false;}}//创建文件夹（如果不存在的话）private static void createFolder(String folder){if(Boolean.TRUE.equals(folderCreateFlag.get(folder))){return;}File f = new File(folder);boolean exists = f.exists();if(!exists){exists = f.mkdirs();log.info("文件夹[{}]已创建！", folder);}if(exists){folderCreateFlag.put(folder, Boolean.TRUE);}}
}

4.测试

package com.vz.utils.report;import java.util.Arrays;
import java.util.Collections;
import java.util.List;/*** @author visy.wang* @description:* @date 2023/7/25 16:02*/
public class DownTest {public static void main(String[] args) {String url1 = "";String url2 = "";String url3 = "";String url4 = "";String url5 = "";String url6 = "";String url7 = "";String url8 = "";String url9 = "";String url10 = "";List<String> urlList1 = Arrays.asList(url1, url2, url3);List<String> urlList2 = Arrays.asList(url4, url5, url6);List<String> urlList3 = Arrays.asList(url7, url8, url9, url10);ProjectDto project1 = new ProjectDto();project1.setName("项目1");project1.addReport(ReportTypeEnum.SETTLEMENT, urlList1);project1.addReport(ReportTypeEnum.BI, urlList2);ProjectDto project2 = new ProjectDto();project2.setName("项目2");project2.addReport(ReportTypeEnum.DATA_REPORT, urlList3);EnterpriseDto enterprise = new EnterpriseDto();enterprise.setName("企业1");enterprise.addProject(project1);enterprise.addProject(project2);DownUtil.download(Collections.singletonList(enterprise), "第一批次");System.out.println("Finished");}
}