Kafka basis

1. important properties
   1. broker
      1. brocerk.id
      2. log.dirs
      3. host.name
      4. zookeeper.connect
      5. controlled.shutdown.enable=true
      6. auto.leader.rebalance.enable=true
   2. consumer
      1. group.id
      2. zookeeper.connect
      3. fetch.message.max.bytes
      4. topic.id
   3. producer
      1. metadata.broker.list
      2. request.required.acks
      3. producer.type
      4. compression.codec
      5. topic.metadata.refresh.interval.ms
      6. batch.num.messages
      7. topic.id
2. topic-level setting
   1. bin/kafka-topics.sh --zookeeper localhost:2181 --create --topic my-topic --partitions 1 --replication-factor 1 --config max.message.bytes=64000 --config flush.messages=1
   2. bin/kafka-topics.sh --zookeeper localhost:2181 --alter --topic my-topic --config max.message.bytes=128000
   3. bin/kafka-topics.sh --zookeeper localhost:2181 --alter --topic my-topic --deleteConfig max.message.bytes
3. Operations
   1. bin/kafka-topics.sh --zookeeper zk_host:port/chroot --create --topic my_topic_name --partitions 20 --replication-factor 3 --config x=y
   2. bin/kafka-topics.sh --zookeeper zk_host:port/chroot --alter --topic my_topic_name --partitions 40
   3. bin/kafka-topics.sh --zookeeper zk_host:port/chroot --alter --topic my_topic_name --config x=y
   4. bin/kafka-topics.sh --zookeeper zk_host:port/chroot --alter --topic my_topic_name --deleteConfig x
   5. bin/kafka-topics.sh --zookeeper zk_host:port/chroot --delete --topic my_topic_name
   6. bin/kafka-preferred-replica-election.sh --zookeeper zk_host:port/chroot
   7. bin/kafka-run-class.sh kafka.tools.ConsumerOffsetChecker --zkconnect localhost:2181 --group test
4. server production server configuration (from kafka document)

   1. # Replication configurations
      num.replica.fetchers=4
      replica.fetch.max.bytes=1048576
      replica.fetch.wait.max.ms=500
      replica.high.watermark.checkpoint.interval.ms=5000
      replica.socket.timeout.ms=30000
      replica.socket.receive.buffer.bytes=65536
      replica.lag.time.max.ms=10000
      replica.lag.max.messages=4000
      
      controller.socket.timeout.ms=30000
      controller.message.queue.size=10
      
      # Log configuration
      num.partitions=8
      message.max.bytes=1000000
      auto.create.topics.enable=true
      log.index.interval.bytes=4096
      log.index.size.max.bytes=10485760
      log.retention.hours=168
      log.flush.interval.ms=10000
      log.flush.interval.messages=20000
      log.flush.scheduler.interval.ms=2000
      log.roll.hours=168
      log.cleanup.interval.mins=30
      log.segment.bytes=1073741824
      
      # ZK configuration
      zookeeper.connection.timeout.ms=6000
      zookeeper.sync.time.ms=2000
      
      # Socket server configuration
      num.io.threads=8
      num.network.threads=8
      socket.request.max.bytes=104857600
      socket.receive.buffer.bytes=1048576
      socket.send.buffer.bytes=1048576
      queued.max.requests=16
      fetch.purgatory.purge.interval.requests=100
      producer.purgatory.purge.interval.requests=100

      
      
5. Hardware
   
   1. disk throughput
   2. 8x7200 rpm SATA drives
   3. higher RPM SAS drives
6. OS setting
   
   1. file descriptors
   2. max socket buffer size