go 问题记录（日志丢失）

问题描述：

在go程序中，通过执行一个命令启动一个子命令，并通过pipe读取子程序的标准输入和输出，通过scanner默认按行读取，此时如果子程序输出时没有携带’\n’，scanner就不会打印输出，而是会累积到缓存buf上限，最终被丢弃，直到遇到一个\n，然后输出所有的内容，默认buf缓存上限时65536，如果日志打印处还有限制，如glog就限制最大的打印字节数为4096，那么就会导致日志再次丢失。

解决方法：

不适用scanner去按行读取，直接读取管道的内容，然后设置上限，超过时或者遇到’\n’时打印

测试代码：

子程序：

#include <stdio.h>
#include <unistd.h>int main() {
int count = 0;while (1) {fprintf(stderr, "%d", count);count = (count + 1) % 10;usleep(500); // Sleep for 500,000 microseconds (0.5 seconds)
}return 0;
}

主程序：

package mainimport ("bufio""fmt""os/exec""strings""log"
)func main() {cmd := exec.Command("./test")stdout, err := cmd.StdoutPipe()if err != nil {fmt.Println("Error creating StdoutPipe:", err)return}cmd.Stderr = cmd.Stdouterr = cmd.Start()if err != nil {fmt.Println("Error starting command:", err)return}scanner := bufio.NewScanner(stdout)// scanner.Split(bufio.ScanBytes)// buf := ""// for scanner.Scan() {// 	   buf += scanner.Text()//     if strings.Contains(buf, "\n") || len(buf) >= 256 {//         log.Printf("%s", buf)//         buf = ""//     }// }for scanner.Scan() {log.Printf("%s", scanner.Text())}if err := scanner.Err(); err != nil {fmt.Println("Error reading standard output:", err)}err = cmd.Wait()if err != nil {fmt.Println("Error waiting for command to finish:", err)}
}

修改程序：

package mainimport ("bufio""fmt""io""log""os/exec"
)func getReaderSize(rd io.Reader) {b, ok := rd.(*bufio.Reader)if ok {log.Printf("rd size: %d", b.Size())} else {log.Printf("rd is not bufio.Reader")}
}func main() {// Command to executecmd := exec.Command("./test")// Create a pipe to capture the standard output of the commandstdout, err := cmd.StdoutPipe()if err != nil {fmt.Println("Error creating StdoutPipe:", err)return}cmd.Stderr = cmd.Stdout// Start the commanderr = cmd.Start()if err != nil {fmt.Println("Error starting command:", err)return} Create a scanner to read the command's standard output//scanner := bufio.NewScanner(stdout)//scanner.Split(bufio.ScanBytes)// Read and print each line from the output//buf := make([]byte, 256)//bufLen := 0//for scanner.Scan() {//	buf[bufLen] = scanner.Bytes()[0]//	// buf = append(buf, scanner.Bytes()...)//	bufLen += 1//	if buf[bufLen-1] == '\n' || bufLen >= 256 {//		log.Printf("%s", string(buf[:bufLen]))//		bufLen = 0//	}//}// Check for errors in scanning//if err := scanner.Err(); err != nil {//	fmt.Println("Error reading standard output:", err)//}// Create a buffered reader to read from the command's stdoutreader := bufio.NewReaderSize(stdout, 256)getReaderSize(stdout)log.Printf("reader size: %d", reader.Size()) Buffer to store incomplete lines//var incompleteLine []byte// Buffer to read chunks of bytes//chunk := make([]byte, 256)////for {//	// Read a chunk of bytes//	n, err := reader.Read(chunk)//	if err != nil {//		break // Break the loop when an error occurs (e.g., when the command finishes)//	}////	// Process each byte in the chunk//	for i := 0; i < n; i++ {//		b := chunk[i]////		// Check for newline or length exceeding 256//		if b == '\n' || len(incompleteLine) >= 256 {//			// Print the line//			log.Printf("%s", incompleteLine)////			// Reset the incomplete line buffer//			incompleteLine = nil//		} else {//			// Add the byte to the incomplete line buffer//			incompleteLine = append(incompleteLine, b)//		}//	}//}for {s, err := reader.ReadSlice('\n')if err != nil && err != bufio.ErrBufferFull {if len(s) > 0 {log.Printf("reader err but exist data, reader size: %d, read string size: %d, string: %s", reader.Size(), len(s), string(s))}fmt.Println("Error reader ReadString:", err)break // Break the loop when an error occurs (e.g., when the command finishes)}log.Printf("reader size: %d, read string size: %d, string: %s", reader.Size(), len(s), string(s))}// Wait for the command to finisherr = cmd.Wait()if err != nil {fmt.Println("Error waiting for command to finish:", err)}
}

benchmark test:

package mainimport ("strconv""strings""testing"
)func stringTest1() string {var buf stringfor i := 0; i < 256; i++ {buf += strconv.Itoa(i)}return buf
}func stringTest2() string {var buf strings.Builderfor i := 0; i < 256; i++ {buf.Write([]byte(strconv.Itoa(i)))}return buf.String()
}func stringTest3() string {var buf = make([]byte, 0)for i := 0; i < 256; i++ {buf = append(buf, []byte(strconv.Itoa(i))...)}return string(buf)
}func stringTest4() string {var buf = make([]byte, 256)for i := 0; i < 256; i++ {buf[i] = '1'}return string(buf)
}func BenchmarkStringTest1(b *testing.B) {for i := 0; i < b.N; i++ {stringTest1()}
}
func BenchmarkStringTest2(b *testing.B) {for i := 0; i < b.N; i++ {stringTest2()}
}
func BenchmarkStringTest3(b *testing.B) {for i := 0; i < b.N; i++ {stringTest3()}
}
func BenchmarkStringTest4(b *testing.B) {for i := 0; i < b.N; i++ {stringTest4()}
}

benchmark test
cmd:

go test -bench . -benchmem
go test -bench=<function>