远程xml读取解析，将image url下载到本地，延时队列定时删除文件，图片访问路径保存在数据库中

远程xml部分内容

<imagelist name="FY4A AGRI IMG REGI MTCC GLL" tag="FY4A AGRI IMG REGI MTCC GLL"><image time="2023-07-25 22:30 (UTC)" desc="FY4A AGRI IMG REGI MTCC GLL" url="http://img.nsmc.org.cn/PORTAL/FY4/IMG/FY4A/AGRI/IMG/REGI/MTCC/GLL/FY4A-_AGRI--_N_REGI_1047E_L1C_MTCC_MULT_GLL_20230725223000_20230725223417_1000M_V0001.JPG"/><image time="2023-07-25 22:23 (UTC)" desc="FY4A AGRI IMG REGI MTCC GLL" url="http://img.nsmc.org.cn/PORTAL/FY4/IMG/FY4A/AGRI/IMG/REGI/MTCC/GLL/FY4A-_AGRI--_N_REGI_1047E_L1C_MTCC_MULT_GLL_20230725222336_20230725222753_1000M_V0001.JPG"/>
</imagelist>

mq发布端定时任务发送消息

@Component
public class TFYImage {private ISyncFY syncFY;Log log = LogFactory.getLog(TFYImage.class);@Autowiredpublic TFYImage(ISyncFY syncFY){this.syncFY = syncFY;}@PostConstruct@Scheduled(cron = "0 0 * * * *")public void Task1() {log.info("execute FY Image task");String xmlUrl = "xml path";String localPath = "/media/resource/FY4APic/";String type = "FY4A";syncFY.syncFY4AImage(xmlUrl, localPath, type);}
}

mq消费端
1，远程xml读取
2，xml解析，将image中图片url保存在集合中
3，遍历集合，当本地不存在此图片时，下载图片至本地
4，将图片路径传给延时队列，用于稍后删除图片
5，保存自定义图片访问路径等信息到数据库

import com.alibaba.fastjson.JSONObject;
import com.fasterxml.jackson.databind.JsonNode;
import com.sxqx.entity.MQMessage;
import com.sxqx.listener.IMessageReceiver;
import com.sxqx.mapper.remote.xugu1.fy.IFYImageMapper;
import com.sxqx.pojo.FYImageItem;
import com.sxqx.utils.common.ListUtils;
import com.sxqx.utils.dataConverter.JsonConverter;
import com.sxqx.utils.file.FileHelper;
import com.sxqx.utils.mq.MQMessageSender;
import com.sxqx.utils.xml.XMLUtil;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.springframework.amqp.rabbit.annotation.Queue;
import org.springframework.amqp.rabbit.annotation.RabbitHandler;
import org.springframework.amqp.rabbit.annotation.RabbitListener;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import org.w3c.dom.Document;
import org.w3c.dom.Element;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;import javax.xml.xpath.XPath;
import javax.xml.xpath.XPathConstants;
import javax.xml.xpath.XPathExpression;
import javax.xml.xpath.XPathFactory;
import java.io.File;
import java.io.IOException;
import java.text.SimpleDateFormat;
import java.util.*;@Component
public class SyncFYImgReceiver implements IMessageReceiver {private final IFYImageMapper fyImageMapper;private final MQMessageSender mqMessageSender;@Autowiredpublic SyncFYImgReceiver(MQMessageSender mqMessageSender,IFYImageMapper fyImageMapper) {this.mqMessageSender = mqMessageSender;this.fyImageMapper = fyImageMapper;}Log log = LogFactory.getLog(SyncFYImgReceiver.class);@RabbitListener(queuesToDeclare = {@Queue(name = "sxqxgxw_sync_fy_img")})@RabbitHandler@Overridepublic void onMessageReceived(String mqMessageString) {JsonNode jsonNode = JsonConverter.jsonString2JsonNode(mqMessageString);JsonNode msg = jsonNode.findValue("msg");JsonNode JsonNodeParams = msg.findValue("params");Map<String, Object> params = JsonConverter.jsonNode2HashMap(JsonNodeParams);if (params.size() > 0) {String xmlUrl = params.get("xmlUrl").toString();String localPath = params.get("localPath").toString();String type = params.get("type").toString();if(Objects.equals(type,"FY4A")){List<String> imageUrlList = new ArrayList<>();try {//读取xmlDocument document = XMLUtil.readXMLUrl(xmlUrl);// 创建一个XPath对象XPathFactory xPathFactory = XPathFactory.newInstance();XPath xPath = xPathFactory.newXPath();// 使用XPath表达式获取imagelist节点XPathExpression expr = xPath.compile("/imagelist");NodeList nodeList = (NodeList) expr.evaluate(document, XPathConstants.NODESET);// 遍历imagelist节点Node node = nodeList.item(0);if (node.getNodeType() == Node.ELEMENT_NODE) {Element imagelistElement = (Element) node;// 获取name和tag属性值//String name = imagelistElement.getAttribute("name");//String tag = imagelistElement.getAttribute("tag");//System.out.println("name: " + name);//System.out.println("tag: " + tag);// 获取image节点列表NodeList imageNodes = imagelistElement.getElementsByTagName("image");// 遍历image节点//读取xml中image存入imageUrlListfor (int j = 0; j < imageNodes.getLength(); j++) {Node imageNode = imageNodes.item(j);if (imageNode.getNodeType() == Node.ELEMENT_NODE) {Element imageElement = (Element) imageNode;// 获取image节点的time、desc和url属性值//String time = imageElement.getAttribute("time");//String desc = imageElement.getAttribute("desc");String imageUrl = imageElement.getAttribute("url");imageUrlList.add(imageUrl);}}}} catch (Exception e) {e.printStackTrace();}List<FYImageItem> fyImageItems = new ArrayList<>();//下载图片，组装数据for (String imageUrl : imageUrlList) {// 获取最后一个斜杠的位置int slashIndex = imageUrl.lastIndexOf("/");// 截取文件名String fileName = imageUrl.substring(slashIndex + 1);String filePath = localPath;String savePath = filePath + fileName;File file = new File(savePath);if (file.exists()) {continue;}//下载图片try {long start = System.currentTimeMillis();FileHelper.downloadUsingNIO(imageUrl, savePath);long end = System.currentTimeMillis();long timeDifferenceInSeconds = (end - start) / 1000;log.info(savePath + " 下载耗时：" + timeDifferenceInSeconds + "秒");} catch (IOException e) {e.printStackTrace();}if (file.exists() && file.length() > 0) {//组装数据String[] s = fileName.split("_");if (s.length == 13) {try {String product = s[0].replace("-", "");String dt = s[9].substring(0, 14);SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMddHHmmss");Date date = sdf.parse(dt);Calendar bjc = Calendar.getInstance();bjc.setTime(date);bjc.add(Calendar.HOUR_OF_DAY, 8);date = bjc.getTime();FYImageItem fyImageItem = new FYImageItem();fyImageItem.setTime(date);fyImageItem.setPath("http://本地服务器/mediaResource/FY4APic/" + fileName);fyImageItem.setFilename(fileName);fyImageItem.setProduct(product);fyImageItems.add(fyImageItem);} catch (Exception e) {e.printStackTrace();}}}// 清除数据MQMessage mqMessage = new MQMessage();JSONObject jsonObject = new JSONObject();jsonObject.put("filePath", savePath);mqMessage.setMsg(jsonObject);mqMessageSender.send("queue.file_delay_destroy", mqMessage);}//保存到数据库if (fyImageItems.size() > 0) {for (List<FYImageItem> fyImageItems1 : ListUtils.splitList(fyImageItems, 50)) {int rows = fyImageMapper.insertIntoFYImage(fyImageItems1);try {Thread.sleep(50);} catch (InterruptedException ex) {ex.printStackTrace();}}try {Thread.sleep(100);} catch (Exception e) {}} else {log.info("don't need to sync fy4a file and fyImageItems.size() is " + fyImageItems.size());}}}}
}

XMLUtil读取远程xml链接

import org.w3c.dom.Document;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import java.net.HttpURLConnection;
import java.net.URL;public class XMLUtil {public static Document readXMLUrl(String xmlUrl) throws Exception{// 创建一个URL对象URL url = new URL(xmlUrl);// 创建一个DocumentBuilderFactory对象DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();// 创建一个DocumentBuilder对象DocumentBuilder builder = factory.newDocumentBuilder();HttpURLConnection connection = (HttpURLConnection) url.openConnection();// 设置连接超时时间为5秒connection.setConnectTimeout(5000);// 设置读取超时时间为10秒connection.setReadTimeout(10000);// 从URL中读取XML文件并解析Document document = builder.parse(connection.getInputStream());return document;}
}

使用NIO下载文件

package com.sxqx.utils.file;import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.net.URL;
import java.net.URLConnection;
import java.nio.channels.Channels;
import java.nio.channels.ReadableByteChannel;public class FileHelper {public static void deleteFile(File file) {/*File[] listFiles()返回一个抽象路径名数组，这些路径名表示此抽象路径名表示的目录中的文件。*/File[] files = file.listFiles();if (files!=null) {//如果包含文件进行删除操作for (File value : files) {if (value.isFile()) {//删除子文件value.delete();} else if (value.isDirectory()) {//通过递归的方法找到子目录的文件deleteFile(value);}value.delete();//删除子目录}}file.delete();}public static Boolean downloadFile(String urlString, String savePath) {InputStream is = null;FileOutputStream os = null;try {// 构造URLURL url = new URL(urlString);// 打开连接URLConnection con = url.openConnection();// 输入流is = con.getInputStream();// 1K的数据缓冲byte[] bs = new byte[1024];// 读取到的数据长度int len;// 输出的文件流File file = new File(savePath);os = new FileOutputStream(file, true);// 开始读取while ((len = is.read(bs)) != -1) {os.write(bs, 0, len);}return true;} catch (Exception e) {e.printStackTrace();return false;} finally {// 完毕，关闭所有链接try {if (null != os) {os.close();}} catch (IOException e) {e.printStackTrace();}try {if (null != is) {is.close();}} catch (IOException e) {e.printStackTrace();}}//从微信下载图片时如果没有id对应的图片则下载一个空图片，不会存在返回为null的情况}/*** 使用NIO下载文件* @param urlStr* @param file* @throws IOException*/public static void downloadUsingNIO(String urlStr, String file) throws IOException {URL url = new URL(urlStr);ReadableByteChannel rbc = Channels.newChannel(url.openStream());FileOutputStream fos = new FileOutputStream(file);fos.getChannel().transferFrom(rbc, 0, Long.MAX_VALUE);fos.close();rbc.close();}
}