[字符编码]windwos下使用libiconv转换编码格式(二)

在http://t.csdnimg.cn/PLUuz笔记中实现了常用编码格式转换的功能,但这还是一个demo。因为代码中向libiconv库函数传递的字符串是存放在堆空间中的(我也是从网上找例子测试,是否一定要开辟堆空间存放还有待考证),如果一次性转换的字节数很巨大的话,就会导致内存空间不足,进而引发功能异常。

所以,对于需要大量转换的数据,应该采取分段多次转换的方法。

经过观察,有的编码格式每个字符对应的字节是固定的,这样分段是容易的。比如GB2312格式,一个字符占两个字节,那么每次处理的字节数就是2的整倍数即可。

除了上面说的字节数固定的情况,还有向utf8这种字符字节数会变化的情况,这种转换则需要复杂些的处理。

#include <iostream>#include <fstream>   #include <string>  #include <bitset> #include "iconv.h" //包函libiconv库头文件//导入libiconv库#pragma comment(lib,"libiconv.lib")bool readfile(const std::string& _filepath, std::string& _filecontent){bool res = false;std::ifstream file(_filepath);if (!file.is_open()) { // 检查文件是否成功打开  std::cerr << "无法打开文件" << _filepath << std::endl;}else {std::string line;while (std::getline(file, line)) { // 逐行读取文件内容  _filecontent += line;}res = true;}file.close(); // 关闭文件return res;}//使用 libiconv 进行int TransCore(const char* _pdesc, const char* _psrc, const char* _pstrin, size_t ilen, char* _pstrout, size_t* _polen){const char** ppin = &_pstrin;char** ppout = &_pstrout;iconv_t cd = iconv_open(_pdesc, _psrc);if (cd == (iconv_t)-1) {return -1;}memset(_pstrout, 0, *_polen);int res = iconv(cd, ppin, &ilen, ppout, _polen);std::cout <<__FUNCTION__<< " exec res = " << res << std::endl;iconv_close(cd);return res;}/*desc 目标编码字符串src  源编码字符串_strin 转换前内容_strout 转换后内容*/bool TransEncodeFormat(const char* _desc, const char* _src, const std::string& _strin, std::string& _strout) {bool res = false;if (_desc == nullptr || _src == nullptr || _strin.empty()) {std::cout << "入参不符合要求" << std::endl;return res;}size_t  inlen = _strin.length();#ifdef  LOGstd::cout << "需要转换的内容 : [" << _strin << "]" << std::endl;std::cout << "需要转换的字节数 : [" << inlen << "]" << std::endl;#endifsize_t  outlen = inlen * 10;char* tempout = new char[outlen];if (TransCore(_desc, _src, _strin.c_str(), inlen, tempout, &outlen) == 0 && tempout != nullptr) {res = true;}#ifdef  LOGstd::cout << "转换后的内容 : [" << tempout << "]" << std::endl;#endifstd::string temp(tempout);_strout = tempout;delete[] tempout;tempout = nullptr;return res;}/*描述   :  在_strin字符串是正确的utf8格式的情况下,分段将utf8字符转换成其他编码格式内容_desc  :  目标编码格式_strin :  被转换的uft8字符串内容_strout:  转换后字符串内容_segnum:  一段字符串字节个数,默认是100字节返回值 :  true 转换成功  false转换失败*/bool SegmentTransUtf8ToOther(const char* _desc, const std::string& _strin, std::string& _strout, const int& _segnum = 100) {const char* _src = "UTF-8";size_t _transcounter = 0;if (_strin.size() == 0) {//没有内容就返回_strout.clear();return true;}if (_segnum <= 0) {return false;}if (_strin.size() <= _segnum) {//字符串小于等于_segnumstd::cout << "第" << ++_transcounter << "段转换" <<",转换字节数"<< _strin.size() << std::endl;if (TransEncodeFormat(_desc, _src, _strin, _strout) == false) {return false;}}else {//字符串大于_segnumint leftpos = 0;                //左边界位置int endpos = _strin.size() - 1; //  结束位置while (leftpos <= endpos) {    int rightpos = 0;//右边界位置int remainingbytes = endpos - leftpos + 1;   //左边界到结束剩余的字节数std::string outemp;if (remainingbytes <= _segnum) {//剩余字节数小于_segnumrightpos = endpos;std::string temp = _strin.substr(leftpos, remainingbytes); std::cout << "第" << ++_transcounter << "段转换" << ",转换字节数" << temp.size() << std::endl;if (TransEncodeFormat(_desc, _src, temp, outemp) == false) {return false;}_strout += outemp;} else {rightpos = leftpos + (_segnum - 1);const char lastbyte = _strin[rightpos];//通过要截取的最后一个字节 判断截取字符串是否完整if (((char)(lastbyte | 0x7f) == (char)0x7f) && ((char)(lastbyte & 0x00) == (char)0x00)) {//最后一个字节是 0XXX XXXXstd::string temp = _strin.substr(leftpos, rightpos - leftpos + 1);std::cout << "第" << ++_transcounter << "段转换" << ",转换字节数" << temp.size() << std::endl;if (TransEncodeFormat(_desc, _src, temp, outemp) == false) {return false;}_strout += outemp;} else if (((char)(lastbyte | 0xbf) == (char)0xbf) && ((char)(lastbyte & 0x80) == (char)0x80)) {//最后一个字节是 10XX XXXXwhile (1) {rightpos = rightpos + 1;if (rightpos > endpos) {//判断rightpos是否超出边界return false;}const char lastbytetemp = _strin[rightpos];if (((char)(lastbytetemp | 0xbf) == (char)0xbf) && ((char)(lastbytetemp & 0x80) == (char)0x80)) {//最后一个字节是 10XX XXXX}else {//最后一个字节不是 10XX XXXX  那么就少截取一个并跳出while循环rightpos = rightpos - 1;break;}}//whileif (rightpos < 0 || rightpos < leftpos) {//rightpos 上面进行了减法所以判断一下return false;}std::string temp = _strin.substr(leftpos, rightpos - leftpos + 1);std::cout << "第" << ++_transcounter << "段转换" << ",转换字节数" << temp.size() << std::endl;if (TransEncodeFormat(_desc, _src, temp, outemp) == false) {return  false;}_strout += outemp;} else if (((char)(lastbyte | 0xdf) == (char)0xdf) && ((char)(lastbyte & 0xc0) == (char)0xc0)) {//最后一个字节是 110X XXXXrightpos = rightpos + 1;if (rightpos > endpos) {//判断rightpos是否超出边界return false;}std::string temp = _strin.substr(leftpos, rightpos - leftpos + 1);std::cout << "第" << ++_transcounter << "段转换" << ",转换字节数" << temp.size() << std::endl;if (TransEncodeFormat(_desc, _src, temp, outemp) == false) {return false;}_strout += outemp;} else if (((char)(lastbyte | 0xef) == (char)0xef) && ((char)(lastbyte & 0xe0) == (char)0xe0)) {//最后一个字节是 1110 XXXXrightpos = rightpos + 2;if (rightpos > endpos) {//判断rightpos是否超出边界return false;}std::string temp = _strin.substr(leftpos, rightpos - leftpos + 1);std::cout << "第" << ++_transcounter << "段转换" << ",转换字节数" << temp.size() << std::endl;if (TransEncodeFormat(_desc, _src, temp, outemp) == false) {return false;}_strout += outemp;} else if (((char)(lastbyte | 0xf7) == (char)0xf7) && ((char)(lastbyte & 0xf0) == (char)0xf0)) {//最后一个字节是 1111 0XXXrightpos = rightpos + 3;if (rightpos > endpos) {//判断rightpos是否超出边界return false;}std::string temp = _strin.substr(leftpos, rightpos - leftpos + 1);std::cout << "第" << ++_transcounter << "段转换" << ",转换字节数" << temp.size() << std::endl;if (TransEncodeFormat(_desc, _src, temp, outemp) == false) {return false;}_strout += outemp;}}leftpos = rightpos + 1;}}std::cout << __FUNCTION__ << " exec success" << std::endl;return true;}int main(int argc, char* argv[]){{std::string filecontent;std::string transcontent;std::string gbkfilepath = "./test-file/utf-8.txt";readfile(gbkfilepath, filecontent);std::cout << "  ./test-file/utf-8.txt 内容字节数 = " << filecontent.size() << std::endl;bool res = SegmentTransUtf8ToOther("GBK", filecontent, transcontent, 1000);std::cout << " transcontent 内容字节数 = " << transcontent.size() << std::endl;std::cout << " transcontent GBK 内容[" << transcontent <<"]" << std::endl;std::cout << "====================================================" << std::endl;}{std::string filecontent;std::string transcontent;std::string gbkfilepath = "./test-file/utf-8.txt";readfile(gbkfilepath, filecontent);std::cout << "  ./test-file/utf-8.txt 内容字节数 = " << filecontent.size() << std::endl;bool res = SegmentTransUtf8ToOther("GB18030", filecontent, transcontent, 1000);std::cout << " transcontent 内容字节数 = " << transcontent.size() << std::endl;std::cout << " transcontent GB18030 内容[" << transcontent << "]" << std::endl;std::cout << "====================================================" << std::endl;}{std::string filecontent;std::string transcontent;std::string gbkfilepath = "./test-file/utf-8.txt";readfile(gbkfilepath, filecontent);std::cout << "  ./test-file/utf-8.txt 内容字节数 = " << filecontent.size() << std::endl;bool res = SegmentTransUtf8ToOther("GB2312", filecontent, transcontent, 1000);std::cout << " transcontent 内容字节数 = " << transcontent.size() << std::endl;std::cout << " transcontent GB2312 内容[" << transcontent << "]" << std::endl;std::cout << "====================================================" << std::endl;}return 0;}