Loading...
Loading...
Apache Kafka 消息队列技能。覆盖生产者 acks=all/1/0 选择决策、幂等性(enable.idempotence + acks=all)配置、消费者手动提交 offset(enable.auto.commit=false)、DLT死信处理(@RetryableTopic/@DltHandler)、@KafkaListener + 异常处理器(DefaultErrorHandler)、顺序消息(同一Key发同一分区)、事务消息。 纠正 LLM:acks=0 丢消息、auto.commit 漏消息、不处理重复消费、不配置 DLT。
npx skill4agent add full-stack-skills/java-skills kafka-patterns| acks值 | 可靠性 | 延迟 | 适用场景 |
|---|---|---|---|
| ⭐⭐⭐ 最高 | 高 | 核心业务(订单/支付/交易) ✅ |
| ⭐⭐ 中 | 中 | 默认值,一般业务 |
| ⭐ 最低 | 低 | 日志/监控/不重要的指标 |
核心业务必须:+acks=all(Broker配置) +min.insync.replicas=2replication.factor=3
| # | 错误 | 正确做法 |
|---|---|---|
| 1 | acks=0 或 acks=1 丢消息 | 核心业务 acks=all + min.insync.replicas=2 |
| 2 | enable.auto.commit=true(自动提交offset) | 手动提交,处理完再 acknowledge |
| 3 | 消费者不处理重复消费 | 业务层幂等(唯一键+数据库去重) |
| 4 | 不配置DLT死信,异常消息无限重试 | DefaultErrorHandler + DeadLetterPublishingRecoverer |
| 5 | 所有消息发到同一个分区 | 用Key路由到分区,同一Key保证顺序 |
| 6 | 同步发送 | 异步发送 + callback 处理异常 |
| 7 | 不配置 enable.idempotence=true | 幂等生产防止重试导致重复消息 |
| 8 | 监听器内异常不处理 | 配置 ErrorHandler,异常→DLT |
@Bean
public ProducerFactory<String, Object> producerFactory() {
Map<String, Object> config = new HashMap<>();
config.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9092");
config.put(ProducerConfig.ACKS_CONFIG, "all"); // ✅ 所有副本确认
config.put(ProducerConfig.RETRIES_CONFIG, 3); // 重试3次
config.put(ProducerConfig.ENABLE_IDEMPOTENCE_CONFIG, true); // ✅ 幂等
config.put(ProducerConfig.MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION, 5);
config.put(ProducerConfig.DELIVERY_TIMEOUT_MS_CONFIG, 120_000); // 投递超时
config.put(ProducerConfig.COMPRESSION_TYPE_CONFIG, "zstd"); // 压缩
config.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class);
config.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, JsonSerializer.class);
return new DefaultKafkaProducerFactory<>(config);
}@Bean
public ConsumerFactory<String, Object> consumerFactory() {
Map<String, Object> config = new HashMap<>();
config.put(ConsumerConfig.GROUP_ID_CONFIG, "order-group");
config.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest"); // 从最早未消费开始
config.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, false); // ✅ 手动提交
config.put(ConsumerConfig.MAX_POLL_RECORDS_CONFIG, 500);
config.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);
config.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, JsonDeserializer.class);
return new DefaultKafkaConsumerFactory<>(config);
}
// ✅ DLT 配置
@Bean
public ConcurrentKafkaListenerContainerFactory<String, Object> kafkaListenerContainerFactory(
ConsumerFactory<String, Object> cf, KafkaTemplate<String, Object> kt) {
ConcurrentKafkaListenerContainerFactory<String, Object> factory =
new ConcurrentKafkaListenerContainerFactory<>();
factory.setConsumerFactory(cf);
// 重试3次,每次间隔1秒,之后进DLT
factory.setCommonErrorHandler(new DefaultErrorHandler(
new DeadLetterPublishingRecoverer(kt),
new FixedBackOff(1000L, 3L)));
return factory;
}// 方式一:直接 @KafkaListener + DLT ErrorHandler
@KafkaListener(topics = "order-topic", groupId = "order-group")
public void listen(ConsumerRecord<String, OrderEvent> record, Acknowledgment ack) {
try {
processOrder(record.value());
ack.acknowledge(); // ✅ 手动提交 offset
} catch (Exception e) {
log.error("处理订单失败: key={}", record.key(), e);
throw e; // 抛出 → ErrorHandler → 重试 → DLT
}
}
// 方式二:@RetryableTopic 自动重试 + DLT
@RetryableTopic(
attempts = "4",
backoff = @Backoff(delay = 1000, multiplier = 2.0, maxDelay = 10000),
kafkaTemplate = "kafkaTemplate")
@KafkaListener(topics = "order-topic", groupId = "order-group")
public void processOrder(OrderEvent event) {
// 业务逻辑,失败自动重试(1s→2s→4s→...)
}
@DltHandler
public void handleDlt(OrderEvent event) {
log.error("消息进入死信队列: {}", event);
// 人工处理或告警
}@Service
public class OrderEventPublisher {
private final KafkaTemplate<String, Object> kafkaTemplate;
public void publishOrderCreated(OrderEvent event) {
String key = String.valueOf(event.getOrderId()); // ✅ 同一订单→同一分区
ListenableFuture<SendResult<String, Object>> future =
kafkaTemplate.send("order-topic", key, event);
// ✅ 异步处理发送结果
future.addCallback(result -> log.debug("发送成功: {}", result.getRecordMetadata().offset()),
ex -> log.error("发送失败: key={}", key, ex));
}
}@KafkaListener(topics = "order-topic")
public void listen(OrderEvent event) {
// ✅ 业务幂等:用事件ID去重
String eventId = event.getEventId();
Boolean existed = redisTemplate.opsForValue().setIfAbsent(
"kafka:dedup:" + eventId, "1", Duration.ofHours(24));
if (Boolean.FALSE.equals(existed)) {
log.debug("重复事件跳过: {}", eventId);
return;
}
// 处理业务
processOrder(event);
}