# -*- coding: utf-8 -*-

'''
callback也是保證分區(qū)有序的, 比如2條消息, a先發(fā)送, b后發(fā)送, 對(duì)于同一個(gè)分區(qū), 那么會(huì)先回調(diào)a的callback, 再回調(diào)b的callback
'''

import json
from kafka import KafkaProducer

topic = 'demo'


def on_send_success(record_metadata):
  print(record_metadata.topic)
  print(record_metadata.partition)
  print(record_metadata.offset)


def on_send_error(excp):
  print('I am an errback: {}'.format(excp))


def main():
  producer = KafkaProducer(
    bootstrap_servers='localhost:9092'
  )
  producer.send(topic, value=b'{"test_msg":"hello world"}').add_callback(on_send_success).add_callback(
    on_send_error)
  # close() 方法會(huì)阻塞等待之前所有的發(fā)送請(qǐng)求完成后再關(guān)閉 KafkaProducer
  producer.close()


def main2():
  '''
  發(fā)送json格式消息
  :return:
  '''
  producer = KafkaProducer(
    bootstrap_servers='localhost:9092',
    value_serializer=lambda m: json.dumps(m).encode('utf-8')
  )
  producer.send(topic, value={"test_msg": "hello world"}).add_callback(on_send_success).add_callback(
    on_send_error)
  # close() 方法會(huì)阻塞等待之前所有的發(fā)送請(qǐng)求完成后再關(guān)閉 KafkaProducer
  producer.close()
if __name__ == '__main__':
  # main()
  main2()

# -*- coding: utf-8 -*-

'''
消費(fèi)者: group_id=None
'''
from kafka import KafkaConsumer
topic = 'demo'
def main():
  consumer = KafkaConsumer(
    topic,
    bootstrap_servers='localhost:9092',
    auto_offset_reset='latest',
    # auto_offset_reset='earliest',
  )
  for msg in consumer:
    print(msg)
    print(msg.value)
  consumer.close()
if __name__ == '__main__':
  main()

# -*- coding: utf-8 -*-

'''
消費(fèi)者: 指定group_id
'''

from kafka import KafkaConsumer

topic = 'demo'
group_id = 'test_id'


def main():
  consumer = KafkaConsumer(
    topic,
    bootstrap_servers='localhost:9092',
    auto_offset_reset='latest',
    group_id=group_id,

  )
  while True:
    try:
      # return a dict
      batch_msgs = consumer.poll(timeout_ms=1000, max_records=2)
      if not batch_msgs:
        continue
      '''
      {TopicPartition(topic='demo', partition=0): [ConsumerRecord(topic='demo', partition=0, offset=42, timestamp=1576425111411, timestamp_type=0, key=None, value=b'74', headers=[], checksum=None, serialized_key_size=-1, serialized_value_size=2, serialized_header_size=-1)]}
      '''
      for tp, msgs in batch_msgs.items():
        print('topic: {}, partition: {} receive length: '.format(tp.topic, tp.partition, len(msgs)))
        for msg in msgs:
          print(msg.value)
    except KeyboardInterrupt:
      break

  consumer.close()


if __name__ == '__main__':
  main()

# -*- coding: utf-8 -*-

'''
producer performance

environment:
  mac
  python3.7
  broker 1
  partition 2
'''

import json
import time
from kafka import KafkaProducer

topic = 'demo'
nums = 1000000


def main():
  producer = KafkaProducer(
    bootstrap_servers='localhost:9092',
    value_serializer=lambda m: json.dumps(m).encode('utf-8')
  )
  st = time.time()
  cnt = 0
  for _ in range(nums):
    producer.send(topic, value=_)
    cnt += 1
    if cnt % 10000 == 0:
      print(cnt)

  producer.flush()

  et = time.time()
  cost_time = et - st
  print('send nums: {}, cost time: {}, rate: {}/s'.format(nums, cost_time, nums // cost_time))


if __name__ == '__main__':
  main()

'''
send nums: 1000000, cost time: 61.89236712455749, rate: 16157.0/s
send nums: 1000000, cost time: 61.29534196853638, rate: 16314.0/s
'''

# -*- coding: utf-8 -*-

'''
consumer performance
'''

import time
from kafka import KafkaConsumer

topic = 'demo'
group_id = 'test_id'


def main1():
  nums = 0
  st = time.time()

  consumer = KafkaConsumer(
    topic,
    bootstrap_servers='localhost:9092',
    auto_offset_reset='latest',
    group_id=group_id
  )
  for msg in consumer:
    nums += 1
    if nums >= 500000:
      break
  consumer.close()

  et = time.time()
  cost_time = et - st
  print('one_by_one: consume nums: {}, cost time: {}, rate: {}/s'.format(nums, cost_time, nums // cost_time))


def main2():
  nums = 0
  st = time.time()

  consumer = KafkaConsumer(
    topic,
    bootstrap_servers='localhost:9092',
    auto_offset_reset='latest',
    group_id=group_id
  )
  running = True
  batch_pool_nums = 1
  while running:
    batch_msgs = consumer.poll(timeout_ms=1000, max_records=batch_pool_nums)
    if not batch_msgs:
      continue
    for tp, msgs in batch_msgs.items():
      nums += len(msgs)
      if nums >= 500000:
        running = False
        break

  consumer.close()

  et = time.time()
  cost_time = et - st
  print('batch_pool: max_records: {} consume nums: {}, cost time: {}, rate: {}/s'.format(batch_pool_nums, nums,
                                              cost_time,
                                              nums // cost_time))


if __name__ == '__main__':
  # main1()
  main2()

'''
one_by_one: consume nums: 500000, cost time: 8.018627166748047, rate: 62354.0/s
one_by_one: consume nums: 500000, cost time: 7.698841094970703, rate: 64944.0/s


batch_pool: max_records: 1 consume nums: 500000, cost time: 17.975456953048706, rate: 27815.0/s
batch_pool: max_records: 1 consume nums: 500000, cost time: 16.711708784103394, rate: 29919.0/s

batch_pool: max_records: 500 consume nums: 500369, cost time: 6.654940843582153, rate: 75187.0/s
batch_pool: max_records: 500 consume nums: 500183, cost time: 6.854053258895874, rate: 72976.0/s

batch_pool: max_records: 1000 consume nums: 500485, cost time: 6.504687070846558, rate: 76942.0/s
batch_pool: max_records: 1000 consume nums: 500775, cost time: 7.047331809997559, rate: 71058.0/s
'''