一个parquet-go例子

一个parquet-go例子

使用go读写parquet，使用到了框架github.com/xitongsys/parquet-go

代码:

package mainimport ("log""time""github.com/xitongsys/parquet-go-source/local""github.com/xitongsys/parquet-go/parquet""github.com/xitongsys/parquet-go/reader""github.com/xitongsys/parquet-go/writer"
)type Student struct {Name    string  `parquet:"name=name, type=BYTE_ARRAY, convertedtype=UTF8, encoding=PLAIN_DICTIONARY"`Age     int32   `parquet:"name=age, type=INT32, encoding=PLAIN"`Id      int64   `parquet:"name=id, type=INT64"`Weight  float32 `parquet:"name=weight, type=FLOAT"`Sex     bool    `parquet:"name=sex, type=BOOLEAN"`Day     int32   `parquet:"name=day, type=INT32, convertedtype=DATE"`Ignored int32   //without parquet tag and won't write
}func main() {var err errorfw, err := local.NewLocalFileWriter("flat.parquet")if err != nil {log.Println("Can't create local file", err)return}//writepw, err := writer.NewParquetWriter(fw, new(Student), 4)if err != nil {log.Println("Can't create parquet writer", err)return}pw.RowGroupSize = 128 * 1024 * 1024 //128Mpw.PageSize = 8 * 1024              //8Kpw.CompressionType = parquet.CompressionCodec_SNAPPYnum := 100for i := 0; i < num; i++ {stu := Student{Name:   "StudentName",Age:    int32(20 + i%5),Id:     int64(i),Weight: float32(50.0 + float32(i)*0.1),Sex:    bool(i%2 == 0),Day:    int32(time.Now().Unix() / 3600 / 24),}if err = pw.Write(stu); err != nil {log.Println("Write error", err)}}if err = pw.WriteStop(); err != nil {log.Println("WriteStop error", err)return}log.Println("Write Finished")fw.Close()///readfr, err := local.NewLocalFileReader("flat.parquet")if err != nil {log.Println("Can't open file")return}pr, err := reader.NewParquetReader(fr, new(Student), 4)if err != nil {log.Println("Can't create parquet reader", err)return}num = int(pr.GetNumRows())for i := 0; i < num/10; i++ {if i%2 == 0 {pr.SkipRows(10) //skip 10 rowscontinue}stus := make([]Student, 10) //read 10 rowsif err = pr.Read(&stus); err != nil {log.Println("Read error", err)}log.Println(stus)}pr.ReadStop()fr.Close()}

代码示例展示了如何使用parquet-go库在Go语言中写入和读取Parquet文件。