diff options
| author | Viktor Shlapakov <vshlapakov@gmail.com> | 2015-02-20 11:21:12 +0300 |
|---|---|---|
| committer | Viktor Shlapakov <vshlapakov@gmail.com> | 2015-02-25 10:01:25 +0300 |
| commit | 25ad88cbe68c816cf41ae12d6d6bfc7c2a0926e8 (patch) | |
| tree | f66e3234350d92b70e9f4d4039f80e8833bb243b /kafka | |
| parent | 9ad0be662d388b47aadf04d712f5744add6456e3 (diff) | |
| download | kafka-python-25ad88cbe68c816cf41ae12d6d6bfc7c2a0926e8.tar.gz | |
Correct message keys for async batching mode
Diffstat (limited to 'kafka')
| -rw-r--r-- | kafka/producer/base.py | 4 | ||||
| -rw-r--r-- | kafka/protocol.py | 6 |
2 files changed, 5 insertions, 5 deletions
diff --git a/kafka/producer/base.py b/kafka/producer/base.py index 5b41bc9..4d56134 100644 --- a/kafka/producer/base.py +++ b/kafka/producer/base.py @@ -62,7 +62,7 @@ def _send_upstream(queue, client, codec, batch_time, batch_size, # Adjust the timeout to match the remaining period count -= 1 timeout = send_at - time.time() - msgset[topic_partition].append(msg) + msgset[topic_partition].append((msg, key)) # Send collected requests upstream reqs = [] @@ -191,7 +191,7 @@ class Producer(object): self.queue.put((TopicAndPartition(topic, partition), m, key)) resp = [] else: - messages = create_message_set(msg, self.codec, key) + messages = create_message_set([(m, key) for m in msg], self.codec, key) req = ProduceRequest(topic, partition, messages) try: resp = self.client.send_produce_request([req], acks=self.req_acks, diff --git a/kafka/protocol.py b/kafka/protocol.py index 2a39de6..b34a95d 100644 --- a/kafka/protocol.py +++ b/kafka/protocol.py @@ -559,7 +559,7 @@ def create_gzip_message(payloads, key=None): """ message_set = KafkaProtocol._encode_message_set( - [create_message(payload, key) for payload in payloads]) + [create_message(payload, pl_key) for payload, pl_key in payloads]) gzipped = gzip_encode(message_set) codec = ATTRIBUTE_CODEC_MASK & CODEC_GZIP @@ -580,7 +580,7 @@ def create_snappy_message(payloads, key=None): """ message_set = KafkaProtocol._encode_message_set( - [create_message(payload, key) for payload in payloads]) + [create_message(payload, pl_key) for payload, pl_key in payloads]) snapped = snappy_encode(message_set) codec = ATTRIBUTE_CODEC_MASK & CODEC_SNAPPY @@ -595,7 +595,7 @@ def create_message_set(messages, codec=CODEC_NONE, key=None): return a list containing a single codec-encoded message. """ if codec == CODEC_NONE: - return [create_message(m, key) for m in messages] + return [create_message(m, k) for m, k in messages] elif codec == CODEC_GZIP: return [create_gzip_message(messages, key)] elif codec == CODEC_SNAPPY: |