springboot kafka集成(实现producer和consumer)
本文内容纲要：
本文介绍如何在springboot项目中集成kafka收发message。
1、先解决依赖
springboot相关的依赖我们就不提了，和kafka相关的只依赖一个spring-kafka集成包
<dependency>
<groupId>org.springframework.kafka</groupId>
<artifactId>spring-kafka</artifactId>
<version>1.1.1.RELEASE</version>
</dependency>
这里我们先把配置文件展示一下
#==============kafka===================
kafka.consumer.zookeeper.connect=10.93.21.21:2181
kafka.consumer.servers=10.93.21.21:9092
kafka.consumer.enable.auto.commit=true
kafka.consumer.session.timeout=6000
kafka.consumer.auto.commit.interval=100
kafka.consumer.auto.offset.reset=latest
kafka.consumer.topic=test
kafka.consumer.group.id=test
kafka.consumer.concurrency=10
kafka.producer.servers=10.93.21.21:9092
kafka.producer.retries=0
kafka.producer.batch.size=4096
kafka.producer.linger=1
kafka.producer.buffer.memory=40960
2、Configuration：Kafkaproducer
1）通过@Configuration、@EnableKafka，声明Config并且打开KafkaTemplate能力。
2）通过@Value注入application.properties配置文件中的kafka配置。
3）生成bean，@Bean
packagecom.kangaroo.sentinel.collect.configuration;
importjava.util.HashMap;
importjava.util.Map;
importorg.apache.kafka.clients.producer.ProducerConfig;
importorg.apache.kafka.common.serialization.StringSerializer;
importorg.springframework.beans.factory.annotation.Value;
importorg.springframework.context.annotation.Bean;
importorg.springframework.context.annotation.Configuration;
importorg.springframework.kafka.annotation.EnableKafka;
importorg.springframework.kafka.core.DefaultKafkaProducerFactory;
importorg.springframework.kafka.core.KafkaTemplate;
importorg.springframework.kafka.core.ProducerFactory;
@Configuration
@EnableKafka
publicclassKafkaProducerConfig{
@Value("${kafka.producer.servers}")
privateStringservers;
@Value("${kafka.producer.retries}")
privateintretries;
@Value("${kafka.producer.batch.size}")
privateintbatchSize;
@Value("${kafka.producer.linger}")
privateintlinger;
@Value("${kafka.producer.buffer.memory}")
privateintbufferMemory;
publicMap<String,Object>producerConfigs(){
Map<String,Object>props=newHashMap<>();
props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG,servers);
props.put(ProducerConfig.RETRIES_CONFIG,retries);
props.put(ProducerConfig.BATCH_SIZE_CONFIG,batchSize);
props.put(ProducerConfig.LINGER_MS_CONFIG,linger);
props.put(ProducerConfig.BUFFER_MEMORY_CONFIG,bufferMemory);
props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG,StringSerializer.class);
props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG,StringSerializer.class);
returnprops;
}
publicProducerFactory<String,String>producerFactory(){
returnnewDefaultKafkaProducerFactory<>(producerConfigs());
}
@Bean
publicKafkaTemplate<String,String>kafkaTemplate(){
returnnewKafkaTemplate<String,String>(producerFactory());
}
}
实验我们的producer，写一个Controller。想topic=test，key=key，发送消息message
packagecom.kangaroo.sentinel.collect.controller;
importcom.kangaroo.sentinel.common.response.Response;
importcom.kangaroo.sentinel.common.response.ResultCode;
importorg.slf4j.Logger;
importorg.slf4j.LoggerFactory;
importorg.springframework.beans.factory.annotation.Autowired;
importorg.springframework.kafka.core.KafkaTemplate;
importorg.springframework.web.bind.annotation.*;
importjavax.servlet.http.HttpServletRequest;
importjavax.servlet.http.HttpServletResponse;
@RestController
@RequestMapping("/kafka")
publicclassCollectController{
protectedfinalLoggerlogger=LoggerFactory.getLogger(this.getClass());
@Autowired
privateKafkaTemplatekafkaTemplate;
@RequestMapping(value="/send",method=RequestMethod.GET)
publicResponsesendKafka(HttpServletRequestrequest,HttpServletResponseresponse){
try{
Stringmessage=request.getParameter("message");
logger.info("kafka的消息={}"，message);
kafkaTemplate.send("test","key",message);
logger.info("发送kafka成功.");
returnnewResponse(ResultCode.SUCCESS,"发送kafka成功",null);
}catch(Exceptione){
logger.error("发送kafka失败",e);
returnnewResponse(ResultCode.EXCEPTION,"发送kafka失败",null);
}
}
}
3、configuration：kafkaconsumer
1）通过@Configuration、@EnableKafka，声明Config并且打开KafkaTemplate能力。
2）通过@Value注入application.properties配置文件中的kafka配置。
3）生成bean，@Bean
packagecom.kangaroo.sentinel.collect.configuration;
importorg.apache.kafka.clients.consumer.ConsumerConfig;
importorg.apache.kafka.common.serialization.StringDeserializer;
importorg.springframework.beans.factory.annotation.Value;
importorg.springframework.context.annotation.Bean;
importorg.springframework.context.annotation.Configuration;
importorg.springframework.kafka.annotation.EnableKafka;
importorg.springframework.kafka.config.ConcurrentKafkaListenerContainerFactory;
importorg.springframework.kafka.config.KafkaListenerContainerFactory;
importorg.springframework.kafka.core.ConsumerFactory;
importorg.springframework.kafka.core.DefaultKafkaConsumerFactory;
importorg.springframework.kafka.listener.ConcurrentMessageListenerContainer;
importjava.util.HashMap;
importjava.util.Map;
@Configuration
@EnableKafka
publicclassKafkaConsumerConfig{
@Value("${kafka.consumer.servers}")
privateStringservers;
@Value("${kafka.consumer.enable.auto.commit}")
privatebooleanenableAutoCommit;
@Value("${kafka.consumer.session.timeout}")
privateStringsessionTimeout;
@Value("${kafka.consumer.auto.commit.interval}")
privateStringautoCommitInterval;
@Value("${kafka.consumer.group.id}")
privateStringgroupId;
@Value("${kafka.consumer.auto.offset.reset}")
privateStringautoOffsetReset;
@Value("${kafka.consumer.concurrency}")
privateintconcurrency;
@Bean
publicKafkaListenerContainerFactory<ConcurrentMessageListenerContainer<String,String>>kafkaListenerContainerFactory(){
ConcurrentKafkaListenerContainerFactory<String,String>factory=newConcurrentKafkaListenerContainerFactory<>();
factory.setConsumerFactory(consumerFactory());
factory.setConcurrency(concurrency);
factory.getContainerProperties().setPollTimeout(1500);
returnfactory;
}
publicConsumerFactory<String,String>consumerFactory(){
returnnewDefaultKafkaConsumerFactory<>(consumerConfigs());
}
publicMap<String,Object>consumerConfigs(){
Map<String,Object>propsMap=newHashMap<>();
propsMap.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG,servers);
propsMap.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG,enableAutoCommit);
propsMap.put(ConsumerConfig.AUTO_COMMIT_INTERVAL_MS_CONFIG,autoCommitInterval);
propsMap.put(ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG,sessionTimeout);
propsMap.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG,StringDeserializer.class);
propsMap.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG,StringDeserializer.class);
propsMap.put(ConsumerConfig.GROUP_ID_CONFIG,groupId);
propsMap.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG,autoOffsetReset);
returnpropsMap;
}
@Bean
publicListenerlistener(){
returnnewListener();
}
}
newListener()生成一个bean用来处理从kafka读取的数据。Listener简单的实现demo如下：只是简单的读取并打印key和message值
@KafkaListener中topics属性用于指定kafkatopic名称，topic名称由消息生产者指定，也就是由kafkaTemplate在发送消息时指定。
packagecom.kangaroo.sentinel.collect.configuration;
importorg.apache.kafka.clients.consumer.ConsumerRecord;
importorg.slf4j.Logger;
importorg.slf4j.LoggerFactory;
importorg.springframework.kafka.annotation.KafkaListener;
publicclassListener{
protectedfinalLoggerlogger=LoggerFactory.getLogger(this.getClass());
@KafkaListener(topics={"test"})
publicvoidlisten(ConsumerRecord<?,?>record){
logger.info("kafka的key:"+record.key());
logger.info("kafka的value:"+record.value().toString());
}
}
tips:
1）我没有介绍如何安装配置kafka，配置kafka时最好用完全bind网络ip的方式，而不是localhost或者127.0.0.1
2）最好不要使用kafka自带的zookeeper部署kafka，可能导致访问不通。
3）理论上consumer读取kafka应该是通过zookeeper，但是这里我们用的是kafkaserver的地址，为什么没有深究。
4）定义监听消息配置时，GROUP_ID_CONFIG配置项的值用于指定消费者组的名称，如果同组中存在多个监听器对象则只有一个监听器对象能收到消息。
本文内容总结：
原文链接：https://www.cnblogs.com/kangoroo/p/7353330.html