DataCap 自定义 File 转换器

DataCap 支持自定义 File 转换器，使用者可以编写自己的文件转换器集成到 DataCap 中。该文档主要讲解如何快速集成一个文件转换器到 DataCap 系统中。

该模块我们主要使用到的是 file 模块内的代码，我们本文使用 json 来做示例。

模块基本配置

---

新建项目后在 pom.xml 文件中增加以下内容：

<dependencies><dependency><groupId>org.jetbrains.kotlin</groupId><artifactId>kotlin-reflect</artifactId></dependency><dependency><groupId>com.google.inject</groupId><artifactId>guice</artifactId></dependency><dependency><groupId>ch.qos.logback</groupId><artifactId>logback-classic</artifactId></dependency><dependency><groupId>io.edurt.datacap</groupId><artifactId>datacap-file-spi</artifactId></dependency><dependency><groupId>com.fasterxml.jackson.core</groupId><artifactId>jackson-core</artifactId></dependency>
</dependencies><build><plugins><plugin><groupId>org.jetbrains.dokka</groupId><artifactId>dokka-maven-plugin</artifactId></plugin></plugins>
</build>

我们添加 datacap-file-spi 依赖，这样我们就可以实现集成文件转换器。

Json Module 加载器

---

package io.edurt.datacap.file.jsonimport com.google.inject.multibindings.Multibinder
import io.edurt.datacap.file.File
import io.edurt.datacap.file.FileModuleclass JsonModule : FileModule()
{override fun configure(){Multibinder.newSetBinder(this.binder(), File::class.java).addBinding().to(JsonFile::class.java)}
}

Json File 转换器

---

package io.edurt.datacap.file.jsonimport com.fasterxml.jackson.core.JsonEncoding
import com.fasterxml.jackson.core.JsonFactory
import com.fasterxml.jackson.core.JsonGenerationException
import com.fasterxml.jackson.databind.JsonNode
import com.fasterxml.jackson.databind.ObjectMapper
import com.fasterxml.jackson.databind.node.ObjectNode
import io.edurt.datacap.common.utils.DateUtils
import io.edurt.datacap.file.File
import io.edurt.datacap.file.FileConvert.formatFile
import io.edurt.datacap.file.model.FileRequest
import io.edurt.datacap.file.model.FileResponse
import org.slf4j.LoggerFactory.getLogger
import java.io.BufferedReader
import java.io.IOException
import java.io.InputStreamReader
import java.util.Objects.requireNonNullclass JsonFile : File
{private val log = getLogger(this::class.java)override fun format(request: FileRequest): FileResponse{val response = FileResponse()try{log.info("${name()} format start time [ ${DateUtils.now()} ]")log.info("${name()} format headers start")response.headers = request.headerslog.info("${name()} format headers end")log.info("${name()} format columns start")val mapper = ObjectMapper()val columns = mutableListOf<Any>()request.columns.forEach { column ->val jsonNode = mapper.createObjectNode()for (headerIndex in request.headers.indices){val header = request.headers[headerIndex] as Stringwhen (column){is List<*> -> jsonNode.putPOJO(header, column[headerIndex])else -> jsonNode.putPOJO(header, column)}}columns.add(jsonNode)}response.columns = columnslog.info("${name()} format columns end")log.info("${name()} format end time [ ${DateUtils.now()} ]")response.successful = true}catch (e: IOException){response.successful = falseresponse.message = e.message}return response}override fun formatStream(request: FileRequest): FileResponse{val response = FileResponse()try{requireNonNull("Stream must not be null")log.info("${name()} format stream start time [ ${DateUtils.now()} ]")val mapper = ObjectMapper()request.stream?.let {BufferedReader(InputStreamReader(it, Charsets.UTF_8)).use { reader ->val jsonNode: JsonNode = mapper.readTree(reader)log.info("${name()} format stream json node count [ ${jsonNode.size()} ]")val headers = mutableListOf<Any>()if (jsonNode.isArray && jsonNode.size() > 0){jsonNode[0].fieldNames().forEachRemaining { headers.add(it) }}response.headers = headersval columns = mutableListOf<Any>()if (jsonNode.isArray){jsonNode.elements().forEachRemaining { node ->val column = mutableMapOf<String, Any?>()node.fields().forEachRemaining { field ->column[field.key] = field.value}columns.add(column)}}response.columns = columnsit.close()}}log.info("${name()} format stream end time [ ${DateUtils.now()} ]")response.successful = true}catch (e: IOException){response.successful = falseresponse.message = e.message}return response}override fun writer(request: FileRequest): FileResponse{val response = FileResponse()try{log.info("${name()} writer origin path [ ${request.path} ]")log.info("${name()} writer start time [ ${DateUtils.now()} ]")val file = formatFile(request, name())log.info("${name()} writer file absolute path [ ${file.absolutePath} ]")val factory = JsonFactory()factory.createGenerator(file, JsonEncoding.UTF8).use { generator ->generator.writeStartArray()request.columns.forEach { column ->generator.writeStartObject()for (headerIndex in request.headers.indices){when (column){is List<*> -> generator.writeObjectField(request.headers[headerIndex] as String, column[headerIndex])is ObjectNode ->{generator.codec = ObjectMapper()val header = request.headers[headerIndex] as Stringgenerator.writeObjectField(header, column.get(header))}else -> generator.writeObjectField(request.headers[headerIndex] as String, column)}}generator.writeEndObject()}generator.writeEndArray()}log.info("${name()} writer end time [ ${DateUtils.now()} ]")response.path = file.absolutePathresponse.successful = true}catch (e: IOException){response.successful = falseresponse.message = e.message}catch (e: JsonGenerationException){response.successful = falseresponse.message = e.message}return response}override fun reader(request: FileRequest): FileResponse{val response = FileResponse()try{log.info("${name()} reader origin path [ ${request.path} ]")log.info("${name()} reader start time [ ${DateUtils.now()} ]")val file = formatFile(request, name())log.info("${name()} reader file absolute path [ ${file.absolutePath} ]")val mapper = ObjectMapper()val jsonNode: JsonNode = mapper.readTree(file)log.info("${name()} reader file json node count [ ${jsonNode.size()} ]")log.info("${name()} reader file headers start")val headers = mutableListOf<Any>()if (jsonNode.isArray && jsonNode.size() > 0){jsonNode[0].fieldNames().forEachRemaining { headers.add(it) }}response.headers = headerslog.info("${name()} reader file headers end")log.info("${name()} reader file columns start")val columns = mutableListOf<Any>()if (jsonNode.isArray){jsonNode.elements().forEachRemaining { node ->val column = mutableMapOf<String, Any?>()node.fields().forEachRemaining { field ->column[field.key] = field.value}columns.add(column)}}response.columns = columnslog.info("${name()} reader file columns end")response.successful = true}catch (e: Exception){response.successful = falseresponse.message = e.message}return response}
}

File SPI 加载器

---

在 resources 源目录下添加 META-INF 和 services 目录，格式为 resources/META-INF/services，创建 io.edurt.datacap.file.FileModule 文件，内容如下

io.edurt.datacap.file.json.JsonModule

通过以上内容我们实现了 Json 文件转换器的支持。我们只需要在要使用 Json 文件转换器的地方引用该模块即可。比如我们在 server 模块中使用到该模块，则在 server/pom.xml 文件中增加以下内容

<dependency><groupId>io.edurt.datacap</groupId><artifactId>datacap-file-json</artifactId><version>${project.version}</version>
</dependency>

Json Module 测试

---

package io.edurt.datacap.file.jsonimport com.google.inject.Guice.createInjector
import com.google.inject.Injector
import com.google.inject.Key
import com.google.inject.TypeLiteral
import io.edurt.datacap.file.File
import io.edurt.datacap.file.FileManager
import org.junit.Assert.assertEquals
import org.junit.Testclass JsonModuleTest
{private val injector: Injector = createInjector(FileManager())@Testfun test(){injector.getInstance(Key.get(object : TypeLiteral<Set<File>>(){})).stream().findFirst().ifPresent {assertEquals("Json", it.name())}}
}

Json SPI 测试

---

package io.edurt.datacap.file.jsonimport com.google.inject.Guice.createInjector
import com.google.inject.Injector
import io.edurt.datacap.file.FileFilter
import io.edurt.datacap.file.FileManager
import io.edurt.datacap.file.model.FileRequest
import org.junit.Before
import org.junit.Test
import org.slf4j.LoggerFactory.getLogger
import java.io.File
import java.io.FileInputStream
import kotlin.test.assertTrueclass JsonFileTest
{private val log = getLogger(this::class.java)private val name = "Json"private var injector: Injector? = nullprivate val request: FileRequest = FileRequest()@Beforefun before(){injector = createInjector(FileManager())request.name = "test"request.path = System.getProperty("user.dir")request.headers = listOf("name", "age")val l1 = listOf("Test", 12)val l2 = listOf("Test1", 121)request.columns = listOf(l1, l2)}@Testfun testFormat(){injector?.let { injector ->FileFilter.filter(injector, name).ifPresent { file ->val response = file.format(request)log.info("headers: [ ${response.headers} ]")response.columns.let { columns ->columns.forEachIndexed { index, line ->log.info("index: [ $index ], line: [ $line ]")}}assertTrue {response.successful == true}}}}@Testfun testFormatStream(){injector?.let { injector ->FileFilter.filter(injector, name).ifPresent { file ->request.stream = FileInputStream(File("${System.getProperty("user.dir")}/${request.name}.json"))val response = file.formatStream(request)log.info("headers: [ ${response.headers} ]")response.columns.let { columns ->columns.forEachIndexed { index, line ->log.info("index: [ $index ], line: [ $line ]")}}assertTrue {response.successful == true}}}}@Testfun testWriter(){injector?.let { injector ->FileFilter.filter(injector, name).ifPresent { file ->assertTrue {file.writer(request).successful == true}}}}@Testfun testReader(){injector?.let { injector ->FileFilter.filter(injector, name).ifPresent { file ->val response = file.reader(request)log.info("headers: ${response.headers}")response.columns.forEach {log.info("columns: $it")}assertTrue {response.successful == true}}}}
}