1，什么是序列化和反序列化
当需要将数据存入文件或者通过网络发送出去时，需将数据对象转化为字节流，即对数据序列化，而将字节流还原成对象即反序列化。常用的序列化框架有Apache Avro、Twitter的Thrift和Google的Protobuf。它们之间的显著区别就是Avro支持IDL（接口描述语言）和Json描述schema，而Thrift和Protobuf只支持IDL描述Schema。本文将讨论Kafka中如何利用Apache avro框架序列/反序列化。
2，Apache avro序列化步骤
第一步：准备jar包
从官网下载avro-1.7.7.jar 和 avro-tools-1.7.7.jar两个jar包，放到指定文件目录。下载地址为http://www.trieuvan.com/apache/avro/avro-1.7.7/java/。
第二步：编写Schema描述文件
可以用Json和IDL描述schema，本文以Json描述schema文件。如创建stock.avsc的文本文件，文件内容如下：
{
"namespace": "example.avro",
"type": "record",
"name": "Stock",
"fields": [
{"name": "stockCode", "type": "string"},
{"name": "stockName", "type": "string"},
{"name": "tradeTime", "type": "long"},
{"name": "preClosePrice", "type": "float"},
{"name": "openPrice", "type": "float"},
{"name": "currentPrice", "type": "float"},
{"name": "highPrice", "type": "float"},
{"name": "lowPrice", "type": "float"}
]
}
第三步：自动生成Java 对象类
执行如下命令：
java -jar avro-tools-1.7.7.jar compile schema stock.avsc java .
文中的 . 表示当前目录
第四步：在maven工程中引入下面jar包
<dependency>
<groupId>org.apache.avro</groupId>
<artifactId>avro</artifactId>
<version>1.7.7</version>
</dependency>
第五步：将自动生成的java 对象源码放入到工程中，并注意修改代码中的包路径等。可以利用查找example.avro 替换成自己工程的包路径；
第六步：自定义序列化和反序列化类
序列化类：
package com.bigdata.kafkasender;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.util.Map;
import org.apache.avro.io.BinaryEncoder;
import org.apache.avro.io.DatumWriter;
import org.apache.avro.io.EncoderFactory;
import org.apache.avro.specific.SpecificDatumWriter;
import org.apache.kafka.common.errors.SerializationException;
import org.apache.kafka.common.serialization.Serializer;
public class AvroSerializer implements Serializer<Stock> {
@Override
public void close() {
}
@Override
public void configure(Map<String, ?> arg0, boolean arg1) {
}
@Override
public byte[] serialize(String topic, Stock data) {
if (data == null) {
return null;
}
DatumWriter<Stock> writer = new SpecificDatumWriter<>(data.getSchema());
ByteArrayOutputStream out = new ByteArrayOutputStream();
BinaryEncoder encoder = EncoderFactory.get().directBinaryEncoder(out, null);
try {
writer.write(data, encoder);
} catch (IOException e) {
throw new SerializationException(e.getMessage());
}
return out.toByteArray();
}
}
反序列化类：
package com.bigdata.kafkasender;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.util.Map;
import org.apache.avro.io.BinaryDecoder;
import org.apache.avro.io.DatumReader;
import org.apache.avro.io.DecoderFactory;
import org.apache.avro.specific.SpecificDatumReader;
import org.apache.kafka.common.serialization.Deserializer;
public class AvroDeserializer implements Deserializer<Stock> {
@Override
public void close() {}
@Override
public void configure(Map<String, ?> arg0, boolean arg1) {}
@Override
public Stock deserialize(String topic, byte[] data) {
if(data == null) {
return null;
}
Stock stock = new Stock();
ByteArrayInputStream in = new ByteArrayInputStream(data);
DatumReader<Stock> userDatumReader = new SpecificDatumReader<>(stock.getSchema());
BinaryDecoder decoder = DecoderFactory.get().directBinaryDecoder(in, null);
try {
stock = userDatumReader.read(null, decoder);
} catch (IOException e) {
e.printStackTrace();
}
return stock;
}
}
第七步：编写kafka生产者
package com.bigdata.kafkasender;
import org.apache.kafka.clients.producer.KafkaProducer;
import org.apache.kafka.clients.producer.ProducerRecord;
import org.apache.kafka.clients.producer.RecordMetadata;
import java.util.Properties;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Future;
public class KafkaSenderApplication {
public static void main(String[] args) throws ExecutionException, InterruptedException {
Stock[] stocks = new Stock[100];
for (int i = 0; i < 100; i++) {
stocks[i] = new Stock();
stocks[i].setStockCode(String.valueOf(i));
stocks[i].setStockName("stock" + i);
stocks[i].setTradeTime(System.currentTimeMillis());
stocks[i].setPreClosePrice(100.0F);
stocks[i].setOpenPrice(88.8F);
stocks[i].setCurrentPrice(120.5F);
stocks[i].setHighPrice(300.0F);
stocks[i].setLowPrice(12.4F);
}
Properties props = new Properties();
props.put("bootstrap.servers", "xx.x.x.xx:9094");
props.put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer");
props.put("value.serializer", "com.bigdata.kafkasender.AvroSerializer");
KafkaProducer<String, Stock> producer = new KafkaProducer<>(props);
Future<RecordMetadata> result = null;
for (Stock stock : stocks) {
ProducerRecord<String, Stock> record = new ProducerRecord<>("avro1", stock);
RecordMetadata metadata = producer.send(record).get();
StringBuilder sb = new StringBuilder();
sb.append("stock: ").append(stock.toString()).append(" has been sent successfully!").append("\n")
.append("send to partition ").append(metadata.partition())
.append(", offset = ").append(metadata.offset());
System.out.println(sb.toString());
Thread.sleep(100);
}
producer.close();
}
}
第八步：编写消费者
public static void consumeMessage() {
Properties props = new Properties();
/* 定义kakfa 服务的地址，不需要将所有broker指定上 */
props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "sxxxx:9094");
/* 制定consumer group */
// props.put("group.id", "flume");
props.put("group.id", "avro1");
props.put("auto.offset.reset", "earliest");
/* key的序列化类 */
props.put("key.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
/* value的序列化类 */
props.put("value.deserializer", "com.bigdata.kafkasender.AvroDeserializer");
/* 定义consumer */
KafkaConsumer<String, Stock> consumer = new KafkaConsumer<>(props);
/* 消费者订阅的topic, 可同时订阅多个 */
consumer.subscribe(Arrays.asList("avro1"));
System.out.println("begin consume!");
/* 读取数据，读取超时时间为100ms */
Duration duration = Duration.ofMillis(100);
while (true) {
ConsumerRecords<String, Stock> records = consumer.poll(duration);
for (ConsumerRecord<String, Stock> record : records)
System.out.printf("offset = %d, key = %s, value = %s", record.offset(), record.key(),
record.value().toString() + "\n");
}
}
public static void main(String[] args) {
// produceMessage();
consumeMessage();
}
