多文件夹图片预处理：清除空值、重置大小、分割训练集

→

清理空值 防止出现cannot identify image file

参考Python数据清洗----删除读取失败图片__简单版_python用pil读取图片出错删除掉-CSDN博客

import os
import shutil
import warnings
import cv2
import iofrom PIL import Image
warnings.filterwarnings("error", category=UserWarning)PATH1 = "./dataclean" #总路径
i = 0def is_read_successfully(file):try:imgFile = Image.open(file)return Trueexcept Exception:return Falseif __name__=="__main__":#子文件夹for childPATH in os.listdir(PATH1):#子文件夹路径childPATH = PATH1 + '/'+ str(childPATH)for parent, dirs, files in os.walk(PATH1):for file in files:if not is_read_successfully(os.path.join(parent, file)):print(os.path.join(parent, file))i = i + 1os.remove(os.path.join(parent, file)) print(i)

重置大小 参考python批量修改图片尺寸(含多个文件夹)_python 修改路径下多个子文件下图片尺寸并重新保存-CSDN博客

# -*- coding: utf-8 -*-
import cv2
import matplotlib.pyplot as plt
import os
import re
import sys
from PIL import Image
import string
import numpy as np
PATH1 = 'dataclean'  #总路径
def resizeImage(file,NoResize):image = cv2.imread(file,cv2.IMREAD_COLOR)#如果type(image) == 'NoneType',会报错,导致程序中断,所以这里先跳过这些图片,#并记录下来,结束程序后手动修改(删除)if image is None:NoResize += [str(file)]else:resizeImg = cv2.resize(image,(100,100))#100*100大小cv2.imwrite(file,resizeImg)cv2.waitKey(100)def resizeAll(root):#待修改文件夹fileList = os.listdir(root)currentpath = os.getcwd()   #将当前工作目录修改为待修改文件夹的位置    os.chdir(root)NoResize = []  #记录没被修改的图片for file in fileList:       #遍历文件夹中所有文件file = str(file)resizeImage(file,NoResize)print("---------------------------------------------------")os.chdir(currentpath)       #改回程序运行前的工作目录sys.stdin.flush()       #刷新print('没被修改的图片: ',NoResize)if __name__=="__main__":#子文件夹for childPATH in os.listdir(PATH1):#子文件夹路径childPATH = PATH1 + '/'+ str(childPATH)# print(childPATH)resizeAll(childPATH)print('------修改完成')

划分训练集测试集 参考【深度学习】使用python划分数据集为训练集和验证集和测试集并放在不同的文件夹_深度学习中有没有直接划分训练集、验证集和测试集的函数-CSDN博客

import os
import random
import shutil
from shutil import copy2"""os.listdir会将文件夹下的文件名集合成一个列表并返回"""
def getDir(filepath):pathlist=os.listdir(filepath)return pathlistdef mkTotalDir(data_path):os.makedirs(data_path)dic=['train','test']for i in range(0,2):current_path=data_path+dic[i]+'/'#这个函数用来判断当前路径是否存在，如果存在则创建失败，如果不存在则可以成功创建isExists=os.path.exists(current_path)if not isExists:os.makedirs(current_path)print('successful '+dic[i])else:print('is existed')return
"""传入的参数是n类图像原本的路径，返回的是这个路径下各类图像的名称列表和图像的类别数"""
def getClassesMes(source_path):classes_name_list=getDir(source_path)classes_num=len(classes_name_list)return classes_name_list,classes_numdef mkClassDir(source_path,change_path):classes_name_list,classes_num=getClassesMes(source_path)for i in range(0,classes_num):current_class_path=os.path.join(change_path,classes_name_list[i])isExists=os.path.exists(current_class_path)if not isExists:os.makedirs(current_class_path)print('successful '+classes_name_list[i])else:print('is existed')#source_path:原始多类图像的存放路径
#train_path:训练集图像的存放路径
#validation_path:验证集图像的存放路径D:\RSdata_dir\NWPU-RESISC45\\
#test_path:测试集图像的存放路径def divideTrainValidationTest(source_path,train_path,test_path):"""先获取五类图像的名称列表和类别数目"""classes_name_list,classes_num=getClassesMes(source_path)mkClassDir(source_path,train_path)mkClassDir(source_path,test_path)"""先将一类图像的路径拿出来，将这个路径下所有这类的图片做成列表，使用os.listdir函数，然后再将列表里面的所有图像名进行shuffle就是随机打乱，然后从打乱后的图像中抽7成放入训练集，3成放入测试集的图像名称列表"""for i in range(0,classes_num):source_image_dir=os.listdir(source_path+classes_name_list[i]+'/')random.shuffle(source_image_dir)train_image_list=source_image_dir[0:int(0.7*len(source_image_dir))]test_image_list=source_image_dir[int(0.7*len(source_image_dir)):]for train_image in train_image_list:origins_train_image_path=source_path+classes_name_list[i]+'/'+train_imagenew_train_image_path=train_path+classes_name_list[i]+'/'copy2(origins_train_image_path,new_train_image_path)for test_image in test_image_list:origins_test_image_path=source_path+classes_name_list[i]+'/'+test_imagenew_test_image_path=test_path+classes_name_list[i]+'/'copy2(origins_test_image_path,new_test_image_path)if __name__=='__main__':source_path = './dataclean/'data_path = './data/'      #运行时自动新建的文件夹train_path = './data/train/'test_path = './data/test/'mkTotalDir(data_path)divideTrainValidationTest(source_path, train_path, test_path)