Question

我正在寻找scala kafka代码的改进。为了减少延迟，我应该在消费者和生产者中做什么。这是我从某人那里获得的代码。我知道这段代码并不难。但是我之前从未见过Scala代码，而我才刚刚开始学习kafka。所以我很难找到问题所在。

import java.util.Properties
import org.apache.kafka.clients.producer.{KafkaProducer, ProducerRecord}

import scala.util.Try

class KafkaMessenger(val servers: String, val sender: String) {
  val props = new Properties()
  props.put("bootstrap.servers", servers)
  props.put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer")
  props.put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer")
  props.put("producer.type", "async")

  val producer = new KafkaProducer[String, String](props)

  def send(topic: String, message: Any): Try[Unit] = Try {
    producer.send(new ProducerRecord(topic, message.toString))
  }

  def close(): Unit = producer.close()
}

object KafkaMessenger {
  def apply(host: String, topic: String, sender: String, message: String): Unit = {
    val messenger = new KafkaMessenger(host, sender)
    messenger.send(topic, message)
    messenger.close()
  }
}

这是消费者代码。

import java.util.Properties
import java.util.concurrent.Executors

import com.satreci.g2gs.common.impl.utils.KafkaMessageTypes._
import kafka.admin.AdminUtils
import kafka.consumer._
import kafka.utils.ZkUtils
import org.I0Itec.zkclient.{ZkClient, ZkConnection}
import org.slf4j.LoggerFactory

import scala.language.postfixOps

class KafkaListener(val zookeeper: String,
                    val groupId: String,
                    val topic: String,
                    val handleMessage: ByteArrayMessage => Unit,
                    val workJson: String = ""
                   ) extends AutoCloseable {
  private lazy val logger = LoggerFactory.getLogger(this.getClass)
  val config: ConsumerConfig = createConsumerConfig(zookeeper, groupId)
  val consumer: ConsumerConnector = Consumer.create(config)
  val sessionTimeoutMs: Int = 10 * 1000
  val connectionTimeoutMs: Int = 8 * 1000
  val zkClient: ZkClient = ZkUtils.createZkClient(zookeeper, sessionTimeoutMs, connectionTimeoutMs)
  val zkUtils = new ZkUtils(zkClient, new ZkConnection(zookeeper), false)

  def createConsumerConfig(zookeeper: String, groupId: String): ConsumerConfig = {
    val props = new Properties()
    props.put("zookeeper.connect", zookeeper)
    props.put("group.id", groupId)
    props.put("auto.offset.reset", "smallest")
    props.put("zookeeper.session.timeout.ms", "5000") 
    props.put("zookeeper.sync.time.ms", "200")
    props.put("auto.commit.interval.ms", "1000")
    props.put("partition.assignment.strategy", "roundrobin")
    new ConsumerConfig(props)
  }

  def run(threadCount: Int = 1): Unit = {
    val streams = consumer.createMessageStreamsByFilter(Whitelist(topic), threadCount)

    if (!AdminUtils.topicExists(zkUtils, topic)) {
      AdminUtils.createTopic(zkUtils, topic, 1, 1)
    }

    val executor = Executors.newFixedThreadPool(threadCount)
    for (stream <- streams) {
      executor.submit(new MessageConsumer(stream))
    }
    logger.debug(s"KafkaListener start with ${threadCount}thread (topic=$topic)")
  }

  override def close(): Unit = {
    consumer.shutdown()
    logger.debug(s"$topic Listener close")
  }

  class MessageConsumer(val stream: MessageStream) extends Runnable {
    override def run(): Unit = {
      val it = stream.iterator()
      while (it.hasNext()) {
        val message = it.next().message()
        if (workJson == "") {
          handleMessage(message)
        }
        else {
          val strMessage = new String(message)
          val newMessage = s"$strMessage/#/$workJson"
          val outMessage = newMessage.toCharArray.map(c => c.toByte)
          handleMessage(outMessage)
        }
      }
    }
  }
}

具体来说，我想在发送消息时修改创建KafkaProduce对象的结构。似乎还有许多其他改进可以减少延迟。

Answer 1

增加具有相同组ID的使用者（KafkaListener）实例的数量。它将增加消耗率。最终，您在生产者撰写和消费者之间的时间差将最小化。

我如何减少卡夫卡消费者/生产者的滞后

1 个答案: