红外相机和RGB相机标定：实现两种模态数据融合

1. 前期准备

1. RGB相机：森云智能SG2-IMX390，1个
2. 红外相机：艾睿光电IR-Pilot 640X-32G，1个
3. 红外标定板：https://item.taobao.com/item.htm?_u=jp3fdd12b99&id=644506141871&spm=a1z09.2.0.0.5f822e8dKrxxYI

2.操作步骤

2.1 采集标定数据

两种模态相机均未进行内参标定，如果发现原始图片畸变较大，可以先进行内参标定。数据采集代码如下，加热红外标定板后断电，移动标定板到合适的位置，按下s键，同时保存IR图和RG图

#!/usr/bin/env python3
import cv2 , time
import numpy as npir_dev = "/dev/video6"
rgb_dev = "/dev/video0"
# define a video capture object 
ir_vid = cv2.VideoCapture(ir_dev) 
rgb_vid = cv2.VideoCapture(rgb_dev) count = 0
while(True): # Capture the video frame by frame st_time = time.time()ret, ir_frame = ir_vid.read()# print(f"{time.time() - st_time}") ret, rgb_frame = rgb_vid.read()print(f"{time.time() - st_time}") # Display the resulting frame height, width = ir_frame.shape[:2]#(512,1280)index = [2*i+1 for i in range(width//2)]vis_ir_frame = ir_frame[:,index,:]vis_rgb_frame = cv2.resize(rgb_frame, (640,512))cv2.imshow('IR frame', vis_ir_frame) cv2.imshow('RGB frame', vis_rgb_frame) key = cv2.waitKey(1) & 0xFF if key == ord('q'): breakif key == ord('s'):cv2.imwrite(f"IR_{count}.png", vis_ir_frame)cv2.imwrite(f"RGB_{count}.png", vis_rgb_frame)count += 1# After the loop release the cap object 
ir_vid.release() 
rgb_vid.release() 
# Destroy all the windows 
cv2.destroyAllWindows() 

2.2 进行标定

核心操作是调用opencv函数cv2.findHomography计算两个相机之间的单应性矩阵，代码如下

#!/usr/bin/python
# -*- coding: UTF-8 -*-
import cv2
import numpy as npdef find_chessboard(filename, pattern=(9,8), wind_name="rgb"):# read input imageimg = cv2.imread(filename)# cv2.imshow("raw", img)# img = cv2.undistort(img, camera_matrix, distortion_coefficients)# convert the input image to a grayscalegray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)# Find the chess board cornersret, corners = cv2.findChessboardCorners(gray, pattern, None)# if chessboard corners are detectedif ret == True:# Draw and display the cornersimg = cv2.drawChessboardCorners(img, pattern, corners, ret)#Draw number，打印角点编号，便于确定对应点corners = np.ceil(corners[:,0,:])for i, pt in enumerate(corners): cv2.putText(img, str(i), (int(pt[0]),int(pt[1])), cv2.FONT_HERSHEY_COMPLEX, 0.3, (0,255,0), 1)cv2.imshow(wind_name,img)return cornersreturn Noneif __name__ == '__main__' :idx = 2 #0~71rgb_img = cv2.imread(f"RGB_{idx}.png")t_img = cv2.imread(f"IR_{idx}.png")#chessboard grid nums in rgb ,注意观察，同一块标定板在RGB相机和红外相机中的格子说可能不一样rgb_width, rgb_height = 9, 8rgb_corners = find_chessboard(f"RGB_{idx}.png", (rgb_width, rgb_height), "rgb")#chessboard grid nums in thermal thermal_width, thermal_height = 11, 8t_corners = find_chessboard(f"IR_{idx}.png", (thermal_width, thermal_height), "thermal")if rgb_corners is not None and t_corners is not None:# test the id correspondence between rgb and thermal cornersrgb_idx = 27 #可视化一个点，确认取对应点的过程是否正确row, col = rgb_idx//rgb_width, rgb_idx%rgb_widtht_idx = row*thermal_width + col + 1pt = rgb_corners[rgb_idx]cv2.putText(rgb_img, str(rgb_idx), (int(pt[0]),int(pt[1])), cv2.FONT_HERSHEY_COMPLEX, 0.3, (0,255,0), 1)pt = t_corners[t_idx]cv2.putText(t_img, str(t_idx), (int(pt[0]),int(pt[1])), cv2.FONT_HERSHEY_COMPLEX, 0.3, (0,255,0), 1)cv2.imshow(f"Point {rgb_idx} on rgb", rgb_img)cv2.imshow(f"Point {t_idx} on thermal", t_img)# Calculate Homographysrc_pts = []for rgb_idx in range(len(rgb_corners)):row, col = rgb_idx//9, rgb_idx%9t_idx = row*11+col+1src_pts.append(t_corners[t_idx])h, status = cv2.findHomography(np.array(src_pts)[:,None,:], rgb_corners[:,None,:])np.savetxt("calib.param", h)# Warp source image to destination based on homographyt_warp = cv2.warpPerspective(t_img, h, (640,512), borderValue=(255,255,255))#colorizet_warp = cv2.applyColorMap(t_warp, cv2.COLORMAP_JET)#mix rgb and thermalalpha = 0.5merge = cv2.addWeighted(rgb_img, alpha, t_warp, 1-alpha, gamma=0)cv2.imshow("warp", merge)cv2.waitKey(0)cv2.destroyAllWindows()

运行结果如下，观察红外和RGB图中角点的对应关系，编号已经可视化出来了

同时，也单独画出了1个对应后的点，如下图，可检查映射关系是否找对

最后，融合结果如下图所示：