From d451a8a64d2912ac63c81c2eb4c02cf920952fdb Mon Sep 17 00:00:00 2001 From: Alex-Cook4 Date: Tue, 7 Jul 2015 09:11:47 -0500 Subject: [PATCH 01/14] Test commit (just added spaces) --- LICENSE.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LICENSE.md b/LICENSE.md index 08c7b2c..277eddb 100644 --- a/LICENSE.md +++ b/LICENSE.md @@ -1,4 +1,4 @@ -##License +##License Licensed under the Apache License, Version 2.0 (the "License"); you may not use this except in compliance with the License. From f8cf3af4d7e682cb3d0f6c6847696779094355e6 Mon Sep 17 00:00:00 2001 From: Alex-Cook4 Date: Tue, 7 Jul 2015 09:17:12 -0500 Subject: [PATCH 02/14] KafkaSource2 --- .../messaging/kafka/KafkaSource2.java | 533 ++++++++++++++++++ 1 file changed, 533 insertions(+) create mode 100644 com.ibm.streamsx.messaging/impl/java/src/com/ibm/streamsx/messaging/kafka/KafkaSource2.java diff --git a/com.ibm.streamsx.messaging/impl/java/src/com/ibm/streamsx/messaging/kafka/KafkaSource2.java b/com.ibm.streamsx.messaging/impl/java/src/com/ibm/streamsx/messaging/kafka/KafkaSource2.java new file mode 100644 index 0000000..5b179ed --- /dev/null +++ b/com.ibm.streamsx.messaging/impl/java/src/com/ibm/streamsx/messaging/kafka/KafkaSource2.java @@ -0,0 +1,533 @@ +/* Generated by Streams Studio: May 26, 2015 1:20:31 PM EDT */ +package com.ibm.streamsx.messaging.kafka; + + +import kafka.api.FetchRequest; +import kafka.api.FetchRequestBuilder; +import kafka.api.PartitionOffsetRequestInfo; +import kafka.common.ErrorMapping; +import kafka.common.TopicAndPartition; +import kafka.javaapi.*; +import kafka.javaapi.consumer.SimpleConsumer; +import kafka.message.MessageAndOffset; + +import java.io.File; +import java.io.FileReader; +import java.io.IOException; +import java.net.URI; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Properties; + +import org.apache.log4j.Logger; + +import scala.actors.threadpool.Arrays; + +import com.ibm.streams.operator.AbstractOperator; +import com.ibm.streams.operator.OperatorContext; +import com.ibm.streams.operator.OutputTuple; +import com.ibm.streams.operator.StreamingOutput; +import com.ibm.streams.operator.log4j.TraceLevel; +import com.ibm.streams.operator.model.OutputPortSet; +import com.ibm.streams.operator.model.OutputPortSet.WindowPunctuationOutputMode; +import com.ibm.streams.operator.model.Libraries; +import com.ibm.streams.operator.model.OutputPorts; +import com.ibm.streams.operator.model.Parameter; +import com.ibm.streams.operator.model.PrimitiveOperator; +import com.ibm.streams.operator.state.Checkpoint; +import com.ibm.streams.operator.state.ConsistentRegionContext; +import com.ibm.streams.operator.state.StateHandler; + +@Libraries({ "opt/downloaded/*" }) +@PrimitiveOperator(name="KafkaSource2", namespace="com.ibm.streamsx.messaging.kafka", +description="Java Operator KafkaSource2") +@OutputPorts({@OutputPortSet(description="Port that produces tuples", cardinality=1, optional=false, windowPunctuationOutputMode=WindowPunctuationOutputMode.Generating), @OutputPortSet(description="Optional output ports", optional=true, windowPunctuationOutputMode=WindowPunctuationOutputMode.Generating)}) +public class KafkaSource2 extends AbstractOperator implements StateHandler { + + private static Logger TRACE = Logger.getLogger(KafkaSource2.class.getName()); + private Thread processThread; + static final String OPER_NAME = "KafkaConsumer"; + private ConsistentRegionContext crContext; + private boolean shutdown = false; + private long triggerCount; + private long triggerIteration = 0; + //consumer variables + public SimpleConsumer myConsumer = null; + String a_topic; + String leadBroker; + int a_port; + long readOffset; + long resetReadOffset = -1; + int a_partition; + int so_timeout=10000; + String clientName; + protected String propertiesFile = null; + List m_replicaBrokers = new ArrayList(); + protected Properties properties = new Properties(), + finalProperties = new Properties(); + + + @Override + public synchronized void initialize(OperatorContext context) + throws Exception { + + super.initialize(context); + Logger.getLogger(this.getClass()).trace("Operator " + context.getName() + " initializing in PE: " + context.getPE().getPEId() + " in Job: " + context.getPE().getJobId() ); + crContext = context.getOptionalContext(ConsistentRegionContext.class); + TRACE.log(TraceLevel.TRACE, "Beginning consumer initialization"); + String propFile = getPropertiesFile(); + if (propFile != null) { + finalProperties.load(new FileReader(propFile)); + } + finalProperties.putAll(properties); + if (finalProperties == null || finalProperties.isEmpty()) + throw new Exception( + "Kafka connection properties must be specified."); + TRACE.log(TraceLevel.TRACE, "Final properties: " + finalProperties.toString()); + + //name client + clientName = a_topic + "_partition_" + Integer.toString(a_partition) + "_pe_" + context.getPE().getPEId(); + TRACE.log(TraceLevel.TRACE, "Initializing consumer with clientName: " + clientName); + + String brokerString = finalProperties.getProperty("metadata.broker.list"); + List hostAndPortStrings = Arrays.asList(brokerString.split(",")); + + PartitionMetadata metadata = findLeader(hostAndPortStrings, a_topic, a_partition); + if (metadata == null) { + TRACE.log(TraceLevel.ERROR, "Can't find metadata for Topic and Partition. Exiting."); + shutdown(); + return; + } + if (metadata.leader() == null) { + TRACE.log(TraceLevel.ERROR, "Can't find Leader for Topic and Partition. Exiting."); + shutdown(); + return; + } + + TRACE.log(TraceLevel.INFO, "BrokerString: " + brokerString + " Broker List: " + hostAndPortStrings.toString() + " Leader: " + metadata.leader()); + + leadBroker = metadata.leader().host(); + a_port = metadata.leader().port(); + + myConsumer = new SimpleConsumer(leadBroker, a_port, so_timeout, 64 * 1024, clientName); + TRACE.log(TraceLevel.TRACE, "Consumer initialization complete."); + + readOffset = getLastOffset(myConsumer, a_topic, a_partition, + kafka.api.OffsetRequest.LatestTime(), clientName); + + + /* + * Create the thread for producing tuples. + * The thread is created at initialize time but started. + * The thread will be started by allPortsReady(). + */ + processThread = getOperatorContext().getThreadFactory().newThread( + new Runnable() { + + @Override + public void run() { + try { + produceTuples(); + } catch (Exception e) { + Logger.getLogger(this.getClass()).error("Operator error", e); //$NON-NLS-1$ + } + } + + }); + + /* + * Set the thread not to be a daemon to ensure that the SPL runtime + * will wait for the thread to complete before determining the + * operator is complete. + */ + processThread.setDaemon(false); + } + + /** + * Notification that initialization is complete and all input and output ports + * are connected and ready to receive and submit tuples. + * @throws Exception Operator failure, will cause the enclosing PE to terminate. + */ + @Override + public synchronized void allPortsReady() throws Exception { + OperatorContext context = getOperatorContext(); + Logger.getLogger(this.getClass()).trace("Operator " + context.getName() + " all ports are ready in PE: " + context.getPE().getPEId() + " in Job: " + context.getPE().getJobId() ); + // Start a thread for producing tuples because operator + // implementations must not block and must return control to the caller. + processThread.start(); + } + + + /** + * Submit new tuples to the output stream + * @throws Exception if an error occurs while submitting a tuple + */ + private void produceTuples() throws Exception { + final StreamingOutput out = getOutput(0); + + + OutputTuple tuple = out.newTuple(); + long numRead; + long currentOffset; + FetchResponse fetchResponse; + + + while(!shutdown){ + //TRACE.log(TraceLevel.TRACE, "Iteration through the while loop. ReadOffset: " + Long.toString(readOffset)); + + if (myConsumer == null){ + myConsumer = new SimpleConsumer(leadBroker, a_port, so_timeout, 64*1024, clientName); + } + + FetchRequest req = new FetchRequestBuilder() + .clientId(clientName) + .addFetch(a_topic, a_partition, + readOffset, 100000).build(); +// System.out.println("Fetched a request! ReadOffset: " + readOffset); + //TRACE.log(TraceLevel.TRACE, "FetchRequest:" + req.toString()); + + try{ + fetchResponse = myConsumer.fetch(req); + //TRACE.log(TraceLevel.TRACE, "Fetched new response:" + fetchResponse.toString()); + } catch(Exception e) { + //System.out.println("Catching exception " + e); + TRACE.log(TraceLevel.ERROR, "Fetch error. Lead server cannot be contacted. Exception: " + e.getStackTrace()); + myConsumer.close(); + myConsumer = null; + fetchResponse = null; + leadBroker = findNewLeader(leadBroker, a_topic, a_partition); + } + + + + if (fetchResponse != null) { + + if (fetchResponse.hasError()){ + // Something went wrong! + short code = fetchResponse.errorCode(a_topic, a_partition); + TRACE.log(TraceLevel.ERROR, "Error fetching data from the Broker:" + leadBroker + " Reason: " + code); + + if (code == ErrorMapping.OffsetOutOfRangeCode()) { + // We asked for an invalid offset. This should never happen. + TRACE.log(TraceLevel.ERROR, "Tried to request an invalid offset. Exiting."); + } + myConsumer.close(); + myConsumer = null; + leadBroker = findNewLeader(leadBroker, a_topic, a_partition); + } else { + numRead = 0; + for (MessageAndOffset messageAndOffset : fetchResponse.messageSet( + a_topic, a_partition)) { + // TRACE.log(TraceLevel.INFO, + // "Looping through messageAndOffset."); + + currentOffset = messageAndOffset.offset(); + tuple = out.newTuple(); + if (currentOffset < readOffset) { + TRACE.log(TraceLevel.ERROR, "Found an old offset: " + + currentOffset + " Expecting: " + readOffset); + } + + ByteBuffer messagePayload = messageAndOffset.message() + .payload(); + byte[] messageBytes = new byte[messagePayload.limit()]; + messagePayload.get(messageBytes); + String message = new String(messageBytes, "UTF-8"); + ByteBuffer keyPayload = messageAndOffset.message().key(); + byte[] keyBytes = new byte[keyPayload.limit()]; + keyPayload.get(keyBytes); + String key = new String(keyBytes, "UTF-8"); + + // Set attributes in tuple: + tuple.setString("message", message); + tuple.setString("key", key); + + try { + if (crContext != null) { + crContext.acquirePermit(); + } + + //if there has been a reset, we NEED to get out of this loop and do a new fetch request + if (resetReadOffset != -1){ + readOffset = resetReadOffset; + resetReadOffset = -1; + break; + } + +// System.out.println( +// "Submitting tuple from offset: " + readOffset + " : " + +// message + " key: " + key); + numRead++; + // Submit tuple to output stream + out.submit(tuple); + readOffset = messageAndOffset.nextOffset(); + + if (crContext != null && crContext.isTriggerOperator()) { + triggerIteration++; + if (triggerIteration >= triggerCount) { + // TRACE.log(TraceLevel.INFO, + // "Trigger fired. Making consistent."); + crContext.makeConsistent(); + triggerIteration = 0; + } + } + } catch (Exception e) { + TRACE.log(TraceLevel.ERROR, + "Unexpected exception: " + e.toString()); + } finally { + // release permit when done submitting + if (crContext != null) { + // TRACE.log(TraceLevel.INFO, "Releasing permit."); + crContext.releasePermit(); + } + } + } + if (numRead == 0) { + try { + Thread.sleep(50); + } catch (InterruptedException ie) { + TRACE.log(TraceLevel.ERROR, "Exception while sleeping: " + ie); + } + } + + + } + + + } + } + if (myConsumer != null) myConsumer.close(); + } + + /** + * Shutdown this operator, which will interrupt the thread + * executing the produceTuples() method. + * @throws Exception Operator failure, will cause the enclosing PE to terminate. + */ + public synchronized void shutdown() throws Exception { + shutdown = true; + if (processThread != null) { + processThread.interrupt(); + processThread = null; + } + OperatorContext context = getOperatorContext(); + Logger.getLogger(this.getClass()).trace("Operator " + context.getName() + " shutting down in PE: " + context.getPE().getPEId() + " in Job: " + context.getPE().getJobId() ); + + // Must call super.shutdown() + super.shutdown(); + } + + @Override + public void close() throws IOException { + TRACE.log(TraceLevel.INFO, "StateHandler close"); + } + + @Override + public void checkpoint(Checkpoint checkpoint) throws Exception { +// System.out.println("Checkpoint readOffset " + readOffset); + TRACE.log(TraceLevel.INFO, "Checkpoint " + checkpoint.getSequenceId()); + checkpoint.getOutputStream().writeLong(readOffset); + } + + @Override + public void drain() throws Exception { +// System.out.println("Drain..."); + TRACE.log(TraceLevel.INFO, "Drain..."); + } + @Override + public void reset(Checkpoint checkpoint) throws Exception { +// System.out.println("Reset"); + TRACE.log(TraceLevel.INFO, "Reset to checkpoint " + checkpoint.getSequenceId()); + resetReadOffset = checkpoint.getInputStream().readLong(); + +// System.out.println("Set readOffset to : " + readOffset); + } + @Override + public void resetToInitialState() throws Exception { +// System.out.println("ResetToInitial"); + TRACE.log(TraceLevel.INFO, "Reset to initial state"); + } + + @Override + public void retireCheckpoint(long id) throws Exception { +// System.out.println("Retire checkpoint"); + TRACE.log(TraceLevel.INFO, "Retire checkpoint"); + } + + /** + * + * @param hostAndPortStrings + * @param topic + * @param partition + * @return + */ + private PartitionMetadata findLeader(List hostAndPortStrings, + String topic, int partition) { + PartitionMetadata returnMetaData = null; + loop: for (String seed : hostAndPortStrings) { + SimpleConsumer consumer = null; + try { + URI uri = new URI("my://" + seed); // may throw URISyntaxException + String host = uri.getHost(); + int port = uri.getPort(); + consumer = new SimpleConsumer(host, port, so_timeout, 64 * 1024, + "leaderLookup"); + List topics = Collections.singletonList(topic); + TopicMetadataRequest req = new TopicMetadataRequest(topics); + kafka.javaapi.TopicMetadataResponse resp = consumer.send(req); + + List metaData = resp.topicsMetadata(); + for (TopicMetadata item : metaData) { + for (PartitionMetadata part : item.partitionsMetadata()) { + if (part.partitionId() == partition) { + returnMetaData = part; + break loop; + } + } + } + } catch (Exception e) { + System.out.println("Error communicating with Broker [" + seed + + "] to find Leader for [" + topic + ", " + + partition + "] Reason: " + e); + TRACE.log(TraceLevel.ERROR, "Error communicating with Broker [" + seed + + "] to find Leader for [" + topic + ", " + + partition + "] Reason: " + e); + } finally { + if (consumer != null) + consumer.close(); + } + } + if (returnMetaData != null) { + m_replicaBrokers.clear(); + for (kafka.cluster.Broker replica : returnMetaData.replicas()) { + String rep = replica.host() + ":" + replica.port(); + TRACE.log(TraceLevel.TRACE, "Adding replica: " + rep); + m_replicaBrokers.add(rep); + } + } + return returnMetaData; + } + + /** + * + * @param a_oldLeader + * @param topic + * @param partition + * @return + */ + private String findNewLeader(String a_oldLeader, String topic, + int partition) throws Exception { + for (int i = 0; i < 3; i++) { + boolean goToSleep = false; + PartitionMetadata metadata = findLeader(m_replicaBrokers, + topic, partition); + if (metadata == null) { + goToSleep = true; + } else if (metadata.leader() == null) { + goToSleep = true; + } else if (a_oldLeader.equalsIgnoreCase(metadata.leader().host()) + && i == 0) { + // first time through if the leader hasn't changed give + // ZooKeeper a second to recover + // second time, assume the broker did recover before failover, + // or it was a non-Broker issue + // + goToSleep = true; + } else { + return metadata.leader().host(); + } + if (goToSleep) { + try { + Thread.sleep(1000); + } catch (InterruptedException ie) { + } + } + } + System.out + .println("Unable to find new leader after Broker failure. Exiting"); + TRACE.log(TraceLevel.ERROR, "Can't find Leader for Topic and Partition. Exiting."); + throw new Exception( + "Unable to find new leader after Broker failure. Exiting"); + } + + public static long getLastOffset(SimpleConsumer consumer, String topic, + int partition, long whichTime, String clientName) { + TopicAndPartition topicAndPartition = new TopicAndPartition(topic, + partition); + Map requestInfo = new HashMap(); + requestInfo.put(topicAndPartition, new PartitionOffsetRequestInfo( + whichTime, 1)); + kafka.javaapi.OffsetRequest request = new kafka.javaapi.OffsetRequest( + requestInfo, kafka.api.OffsetRequest.CurrentVersion(), + clientName); + OffsetResponse response = consumer.getOffsetsBefore(request); + + if (response.hasError()) { + TRACE.log(TraceLevel.ERROR, "Error fetching data Offset Data the Broker. Reason: " + + response.errorCode(topic, partition)); + return 0; + } + long[] offsets = response.offsets(topic, partition); + TRACE.log(TraceLevel.TRACE, "Retrieving offsets: " + Arrays.toString(offsets)); + return offsets[0]; + } + + public String getPropertiesFile() { + TRACE.log(TraceLevel.TRACE, "Properties file: " + propertiesFile); + if (propertiesFile == null) return null; + File file = new File(propertiesFile); + + // if the properties file is relative, the path is relative to the application directory + if (!file.isAbsolute()) + { + propertiesFile = getOperatorContext().getPE().getApplicationDirectory().getAbsolutePath() + "/" + propertiesFile; + } + return propertiesFile; + } + + @Parameter(optional = true, description = "Properties file containing kafka properties. Properties file is recommended to be stored in the etc directory. If a relative path is specified, the path is relative to the application directory.") + public void setPropertiesFile(String value) { + this.propertiesFile = value; + } + + @Parameter(name="topic", optional=false, + description="Topic to be subscribed to.") + public void setTopic(String value) { + if(value!=null) + this.a_topic = value; + } + + @Parameter(name="partition", optional=false, + description="Partition to subscribe to.") + public void setPartition(int value) { + this.a_partition = value; + } + + @Parameter(name="triggerCount", optional=true, + description="Number of messages between checkpointing for consistent region. This is only relevant to operator driven checkpointing.") + public void setTriggerCount(int value) { + this.triggerCount = value; + } + + @Parameter(cardinality = -1, optional = true, description = "Specify a Kafka property \\\"key=value\\\" form. " + + "This will override any property specified in the properties file.") + public void setKafkaProperty(List values) { + for (String value : values) { + int idx = value.indexOf("="); + if (idx == -1) + throw new IllegalArgumentException("Invalid property: " + value + + ", not in the key=value format"); + String name = value.substring(0, idx); + String v = value.substring(idx + 1, value.length()); + properties.setProperty(name, v); + } + } + + +} + + From 62e7d8639b87c705ec26898271799587b6d6d253 Mon Sep 17 00:00:00 2001 From: Alex-Cook4 Date: Tue, 7 Jul 2015 09:21:06 -0500 Subject: [PATCH 03/14] Fixing lost kafka folder... --- com.ibm.streamsx.messaging/.gitignore | 1 - .../.toolkit.xml~ | 130 ++++++++++++++++++ .../KafkaConsumer/.KafkaConsumer.xml~ | 80 +++++++++++ .../KafkaConsumer/.KafkaConsumer_16.gif~ | Bin 0 -> 617 bytes .../KafkaConsumer/.KafkaConsumer_32.gif~ | Bin 0 -> 2617 bytes .../KafkaConsumer/KafkaConsumer.xml | 80 +++++++++++ .../KafkaConsumer/KafkaConsumer_16.gif | Bin 0 -> 617 bytes .../KafkaConsumer/KafkaConsumer_32.gif | Bin 0 -> 2617 bytes .../KafkaProducer/.KafkaProducer.xml~ | 83 +++++++++++ .../KafkaProducer/.KafkaProducer_16.gif~ | Bin 0 -> 618 bytes .../KafkaProducer/.KafkaProducer_32.gif~ | Bin 0 -> 2602 bytes .../KafkaProducer/KafkaProducer.xml | 83 +++++++++++ .../KafkaProducer/KafkaProducer_16.gif | Bin 0 -> 618 bytes .../KafkaProducer/KafkaProducer_32.gif | Bin 0 -> 2602 bytes .../KafkaSource2/KafkaSource2.xml | 72 ++++++++++ 15 files changed, 528 insertions(+), 1 deletion(-) create mode 100644 com.ibm.streamsx.messaging/com.ibm.streamsx.messaging.kafka/.toolkit.xml~ create mode 100644 com.ibm.streamsx.messaging/com.ibm.streamsx.messaging.kafka/KafkaConsumer/.KafkaConsumer.xml~ create mode 100644 com.ibm.streamsx.messaging/com.ibm.streamsx.messaging.kafka/KafkaConsumer/.KafkaConsumer_16.gif~ create mode 100644 com.ibm.streamsx.messaging/com.ibm.streamsx.messaging.kafka/KafkaConsumer/.KafkaConsumer_32.gif~ create mode 100644 com.ibm.streamsx.messaging/com.ibm.streamsx.messaging.kafka/KafkaConsumer/KafkaConsumer.xml create mode 100644 com.ibm.streamsx.messaging/com.ibm.streamsx.messaging.kafka/KafkaConsumer/KafkaConsumer_16.gif create mode 100644 com.ibm.streamsx.messaging/com.ibm.streamsx.messaging.kafka/KafkaConsumer/KafkaConsumer_32.gif create mode 100644 com.ibm.streamsx.messaging/com.ibm.streamsx.messaging.kafka/KafkaProducer/.KafkaProducer.xml~ create mode 100644 com.ibm.streamsx.messaging/com.ibm.streamsx.messaging.kafka/KafkaProducer/.KafkaProducer_16.gif~ create mode 100644 com.ibm.streamsx.messaging/com.ibm.streamsx.messaging.kafka/KafkaProducer/.KafkaProducer_32.gif~ create mode 100644 com.ibm.streamsx.messaging/com.ibm.streamsx.messaging.kafka/KafkaProducer/KafkaProducer.xml create mode 100644 com.ibm.streamsx.messaging/com.ibm.streamsx.messaging.kafka/KafkaProducer/KafkaProducer_16.gif create mode 100644 com.ibm.streamsx.messaging/com.ibm.streamsx.messaging.kafka/KafkaProducer/KafkaProducer_32.gif create mode 100644 com.ibm.streamsx.messaging/com.ibm.streamsx.messaging.kafka/KafkaSource2/KafkaSource2.xml diff --git a/com.ibm.streamsx.messaging/.gitignore b/com.ibm.streamsx.messaging/.gitignore index c138ea5..df6330a 100644 --- a/com.ibm.streamsx.messaging/.gitignore +++ b/com.ibm.streamsx.messaging/.gitignore @@ -1,5 +1,4 @@ /output -com.ibm.streamsx.messaging.kafka impl/lib opt/downloaded /target diff --git a/com.ibm.streamsx.messaging/com.ibm.streamsx.messaging.kafka/.toolkit.xml~ b/com.ibm.streamsx.messaging/com.ibm.streamsx.messaging.kafka/.toolkit.xml~ new file mode 100644 index 0000000..c3d1a16 --- /dev/null +++ b/com.ibm.streamsx.messaging/com.ibm.streamsx.messaging.kafka/.toolkit.xml~ @@ -0,0 +1,130 @@ + + + + + Toolkit Description + + + + + + + + + + + + This operator acts as a Kafka producer sending tuples as mesages to a Kafka broker. The broker is assumed to be already configured and running. The incoming stream can have three attributes: topic, key and message. The message is a required attribute. If the key attribute is not specified, the message is used as the key. A topic can be specified as either an input stream attribute or as a parameter. + +**Behavior in a Consistent Region** +This operator can participate in a consistent region. This operator cannot be placed at the start of a consistent region. + + + + + + Command line options passed to the Java Virtual Machine at startup + + + Specify a Kafka property "key=value" form. This will override any property specified in the properties file. + + + Name of the attribute for the key. Default is "key" + + + Name of the attribute for the message. This attribute is required. Default is "message" + + + Properties file containing kafka properties. Properties file is recommended to be stored in the etc directory. If a relative path is specified, the path is relative to the application directory. + + + Topic to be published to. A topic can also be specified as an input stream attribute. + + + Name of the attribute for the topic. Default is "topic" + + + The tuples arriving on this port are expected to contain three attributes "key", "topic" and "message". Out of these "message", is a required attribute. + NonWindowed + + + + This operator acts as a Kafka consumer receiving messages for one or more topics. Note that there may be multiple threads receiving messages depending on the configuration specified. Ordering of messages is not guaranteed. + +**Behavior in a Consistent Region** +This operator cannot be used inside a consistent region. + + + + + + Command line options passed to the Java Virtual Machine at startup + + + Specify a Kafka property "key=value" form. This will override any property specified in the properties file. + + + Name of the attribute for the key. Default is "key" + + + Name of the attribute for the message. This attribute is required. Default is "message" + + + Properties file containing kafka properties. Properties file is recommended to be stored in the etc directory. If a relative path is specified, the path is relative to the application directory. + + + Number of threads per topic. Default is 1. + + + Topic to be subscribed to. + + + Messages received from Kafka are sent on this output port. + + + + Java Operator NewKafkaConsumer + + Command line options passed to the Java Virtual Machine at startup + + + Specify a Kafka property "key=value" form. This will override any property specified in the properties file. + + + Name of the attribute for the key. Default is "key" + + + Name of the attribute for the message. This attribute is required. Default is "message" + + + Properties file containing kafka properties. Properties file is recommended to be stored in the etc directory. If a relative path is specified, the path is relative to the application directory. + + + Number of threads per topic. Default is 1. + + + Topic to be subscribed to. + + + Port that produces tuples + + + Optional output ports + + + + + + + + + + + + + + + + + + diff --git a/com.ibm.streamsx.messaging/com.ibm.streamsx.messaging.kafka/KafkaConsumer/.KafkaConsumer.xml~ b/com.ibm.streamsx.messaging/com.ibm.streamsx.messaging.kafka/KafkaConsumer/.KafkaConsumer.xml~ new file mode 100644 index 0000000..7be8c51 --- /dev/null +++ b/com.ibm.streamsx.messaging/com.ibm.streamsx.messaging.kafka/KafkaConsumer/.KafkaConsumer.xml~ @@ -0,0 +1,80 @@ + + + + + + This operator acts as a Kafka consumer receiving messages for one or more topics. Note that there may be multiple threads receiving messages depending on the configuration specified. Ordering of messages is not guaranteed. + +**Behavior in a Consistent Region** +This operator cannot be used inside a consistent region. + KafkaConsumer_16.gif + KafkaConsumer_32.gif + + + com.ibm.streamsx.messaging.kafka.KafkaSource + + + + Operator class library + + ../../impl/java/bin + ../../opt/downloaded/* + + + + + + + kafkaProperty + Specify a Kafka property "key=value" form. This will override any property specified in the properties file. + true + rstring + -1 + + + keyAttribute + Name of the attribute for the key. Default is "key" + true + rstring + 1 + + + messageAttribute + Name of the attribute for the message. This attribute is required. Default is "message" + true + rstring + 1 + + + propertiesFile + Properties file containing kafka properties. Properties file is recommended to be stored in the etc directory. If a relative path is specified, the path is relative to the application directory. + true + rstring + 1 + + + threadsPerTopic + Number of threads per topic. Default is 1. + true + int32 + 1 + + + topic + Topic to be subscribed to. + false + rstring + -1 + + + + + + Messages received from Kafka are sent on this output port. + Free + 1 + false + + + + \ No newline at end of file diff --git a/com.ibm.streamsx.messaging/com.ibm.streamsx.messaging.kafka/KafkaConsumer/.KafkaConsumer_16.gif~ b/com.ibm.streamsx.messaging/com.ibm.streamsx.messaging.kafka/KafkaConsumer/.KafkaConsumer_16.gif~ new file mode 100644 index 0000000000000000000000000000000000000000..12b21132b368dde4bfeaedc8b710c36907388043 GIT binary patch literal 617 zcmZ?wbhEHb6krfwc$UF%_~O%5d#?X^wZ3A?zDesYUVZeTVdjAoH(uU<`Ss@0PiGhT zT)g}C&hsxfHzn*ie)qwvZw<2!F5Y%&YqiL-9hd889(Z%__@b?sKEJ+q=H~0=JFjeS zQm>hD@YC0yU%&s_cl!SBR^_Vc2d+QASC`wOP8Jv(j9+39P} zb}TqrIc494)fp}Gj!ay8VPl14#ngRQ*Tud(UB9thq-pNq8+#{AS#|pCt=9+6KKSzH z&bJRwA0KO6w)0Bw%5$stTq~Klr+dYjU%&sBP1<{Kg4Lz_@7m@ax%2ql@9!TQ=N-Lp z=WXZ0W3O-R-gx--`h&OjEh*c!ZuZP|=PurT_wf0LscX)(%|B8+VRu?u8Uq8v|NsAi zB;`QyCkrD3Lk5Em$S6>pFt8tM$lw!5X>GHT%joKoNtL!>oXBXIY#ONF?U5Q_%P@~2 zN+~u;H@ZtQ)h;!Xo7>$)J6_91-ZLX1l|@B|lhc?@h>OeHAUND@r$Pt^hXk7*(?upn zBlDdOiv0YEY|gyAPG(YGVmtjcd3cmX3^n4^>_wYWeO*|;vxW-Cgt@ABrJqXBEL`wl zrg7Sgk{ugf_^@a)EAd%uKHM|emEod-N%AuVClgs)xtJ6;930tU2z0s^81 zMX`voiGTqWBy6%Jf`qkUCm=hrqk^Z0#?R?ugJVhpx6$-`GSXpdrY*$wo1VM@f@<3JK znKNg2JRS&wqFYkGC|O91EIGgBVx;ux)2IH?(t!IfgJWb-PZU?~$-2v9b1Qr3bb77W zG2njL&E)c=yrzuOjzEmDrhyOB=2gb@%W3E)i(^K-Beeafjn4ZWj>xfFPlXwF|W!%D=zR<*qrfgqjrsHTs zM2-(I(vp?w1w@Fe+X{R;i})8JW$}`R9AC}GEMIB;Ktr0X*KJwcZQo39pgiVCllVY> zWzX!)ui}E1>`2?^<(=6TJ=qt5z!-T-p>n7(wIIYO^0A_#ap3vSc29*sQ-aYgd; zrofo8;^JZe0OsfCk@^33Y?xnqj7A5k3?T#g?%XZnF_^Bf5Oji_*&GY>yZmA_$Yxof z9SAgt#-qS4Y#VPrZ0CK*p6Tt%B(l)EcY!-aB#}GM9TqY`k-Hm5KoVJ?87!_7OhVSG zZ7dpGP!YOXpep$9Kps#3shLZcleR!;3;);o&)OH*d+}hbEiB-A@R=~8zVRlwgIof(Qhn2^F&1zG&lfSaU0ai3tV5Z71TWINTmnONanbs1%Bom5HS_ zo=9B8?&AoA3=R`sBKO@ zn-iQ&wiD1F@vQ$PaahD5tg7JtQh`F>*j-P<=~qa(vZg9H7q`+9r2yE;4CU$wO=Tbi318|v$76~EL}S5;P&%gbJt zmb{Qji;D^i@}KAB=459{GM{CnKYj8zEmiy|B{}I~;)8_vxY(Hc_oDAc{d_0#_N|DU zH^Reyx_&M6YDjR%aSl{fxzFEC$<<~2gFZ)Vw=@MO(j<%MjhPoO6Y?#qR zW+|{u_3Wzt;{a+NxTLFZlO8AY*SfBEqbW7MBxpHIw>*}9?JAn7=h9w&r0+&3#bBAf zT7b?{bptywbzl7^%M)o!-V!rH9adVb=8v?`1SHjvG@=d~1_x7Vt5Wq1PE|@mjyP^f z(A;Ae9QR-~;YXV&X;!Mk$zxtKxrEw=*jL#;p5KAosl9$|EvXVW;+To8Ofha7F^KdQ`h zwj}7-pLLAJ2G6fQ+-a{%_@+Mf^x+}BbVq-$erPhU2*4V3i_+DfY2GCqY@ zvQ0}k+<)iP)`b4`Xk8X{Qz$txiNY$d9f;=HzOKR+0jpkM*Xe7%z(Uo>ouhm><4c#J z=2oghK1IaA7*J0lQ!5UZs%aO`UiOPc2_dpJdX^6?(X2UcsI3tgE(0RfBS!9R&<|IG z*1dLWRePj?xvCzc9uH_dM5VN)rl3)`)KF{n$*JU!HTlC>gSfU)apZU8A+?}izTJ9X z%HsigZC7h$YKlMhf3L2iH}0;3)?Zo;6zPF(_s2LN#8~~bSDb=izEl?b!_)yf8QAeB D+e$-u literal 0 HcmV?d00001 diff --git a/com.ibm.streamsx.messaging/com.ibm.streamsx.messaging.kafka/KafkaConsumer/KafkaConsumer.xml b/com.ibm.streamsx.messaging/com.ibm.streamsx.messaging.kafka/KafkaConsumer/KafkaConsumer.xml new file mode 100644 index 0000000..7d0d1aa --- /dev/null +++ b/com.ibm.streamsx.messaging/com.ibm.streamsx.messaging.kafka/KafkaConsumer/KafkaConsumer.xml @@ -0,0 +1,80 @@ + + + + + + This operator acts as a Kafka consumer receiving messages for one or more topics. Note that there may be multiple threads receiving messages depending on the configuration specified. Ordering of messages is not guaranteed. + +**Behavior in a Consistent Region** +This operator cannot be used inside a consistent region. + KafkaConsumer_16.gif + KafkaConsumer_32.gif + + + com.ibm.streamsx.messaging.kafka.KafkaSource + + + + Operator class library + + ../../impl/java/bin + ../../opt/downloaded/* + + + + + + + kafkaProperty + Specify a Kafka property "key=value" form. This will override any property specified in the properties file. + true + rstring + -1 + + + keyAttribute + Name of the attribute for the key. Default is "key" + true + rstring + 1 + + + messageAttribute + Name of the attribute for the message. This attribute is required. Default is "message" + true + rstring + 1 + + + propertiesFile + Properties file containing kafka properties. Properties file is recommended to be stored in the etc directory. If a relative path is specified, the path is relative to the application directory. + true + rstring + 1 + + + threadsPerTopic + Number of threads per topic. Default is 1. + true + int32 + 1 + + + topic + Topic to be subscribed to. + false + rstring + -1 + + + + + + Messages received from Kafka are sent on this output port. + Free + 1 + false + + + + \ No newline at end of file diff --git a/com.ibm.streamsx.messaging/com.ibm.streamsx.messaging.kafka/KafkaConsumer/KafkaConsumer_16.gif b/com.ibm.streamsx.messaging/com.ibm.streamsx.messaging.kafka/KafkaConsumer/KafkaConsumer_16.gif new file mode 100644 index 0000000000000000000000000000000000000000..12b21132b368dde4bfeaedc8b710c36907388043 GIT binary patch literal 617 zcmZ?wbhEHb6krfwc$UF%_~O%5d#?X^wZ3A?zDesYUVZeTVdjAoH(uU<`Ss@0PiGhT zT)g}C&hsxfHzn*ie)qwvZw<2!F5Y%&YqiL-9hd889(Z%__@b?sKEJ+q=H~0=JFjeS zQm>hD@YC0yU%&s_cl!SBR^_Vc2d+QASC`wOP8Jv(j9+39P} zb}TqrIc494)fp}Gj!ay8VPl14#ngRQ*Tud(UB9thq-pNq8+#{AS#|pCt=9+6KKSzH z&bJRwA0KO6w)0Bw%5$stTq~Klr+dYjU%&sBP1<{Kg4Lz_@7m@ax%2ql@9!TQ=N-Lp z=WXZ0W3O-R-gx--`h&OjEh*c!ZuZP|=PurT_wf0LscX)(%|B8+VRu?u8Uq8v|NsAi zB;`QyCkrD3Lk5Em$S6>pFt8tM$lw!5X>GHT%joKoNtL!>oXBXIY#ONF?U5Q_%P@~2 zN+~u;H@ZtQ)h;!Xo7>$)J6_91-ZLX1l|@B|lhc?@h>OeHAUND@r$Pt^hXk7*(?upn zBlDdOiv0YEY|gyAPG(YGVmtjcd3cmX3^n4^>_wYWeO*|;vxW-Cgt@ABrJqXBEL`wl zrg7Sgk{ugf_^@a)EAd%uKHM|emEod-N%AuVClgs)xtJ6;930tU2z0s^81 zMX`voiGTqWBy6%Jf`qkUCm=hrqk^Z0#?R?ugJVhpx6$-`GSXpdrY*$wo1VM@f@<3JK znKNg2JRS&wqFYkGC|O91EIGgBVx;ux)2IH?(t!IfgJWb-PZU?~$-2v9b1Qr3bb77W zG2njL&E)c=yrzuOjzEmDrhyOB=2gb@%W3E)i(^K-Beeafjn4ZWj>xfFPlXwF|W!%D=zR<*qrfgqjrsHTs zM2-(I(vp?w1w@Fe+X{R;i})8JW$}`R9AC}GEMIB;Ktr0X*KJwcZQo39pgiVCllVY> zWzX!)ui}E1>`2?^<(=6TJ=qt5z!-T-p>n7(wIIYO^0A_#ap3vSc29*sQ-aYgd; zrofo8;^JZe0OsfCk@^33Y?xnqj7A5k3?T#g?%XZnF_^Bf5Oji_*&GY>yZmA_$Yxof z9SAgt#-qS4Y#VPrZ0CK*p6Tt%B(l)EcY!-aB#}GM9TqY`k-Hm5KoVJ?87!_7OhVSG zZ7dpGP!YOXpep$9Kps#3shLZcleR!;3;);o&)OH*d+}hbEiB-A@R=~8zVRlwgIof(Qhn2^F&1zG&lfSaU0ai3tV5Z71TWINTmnONanbs1%Bom5HS_ zo=9B8?&AoA3=R`sBKO@ zn-iQ&wiD1F@vQ$PaahD5tg7JtQh`F>*j-P<=~qa(vZg9H7q`+9r2yE;4CU$wO=Tbi318|v$76~EL}S5;P&%gbJt zmb{Qji;D^i@}KAB=459{GM{CnKYj8zEmiy|B{}I~;)8_vxY(Hc_oDAc{d_0#_N|DU zH^Reyx_&M6YDjR%aSl{fxzFEC$<<~2gFZ)Vw=@MO(j<%MjhPoO6Y?#qR zW+|{u_3Wzt;{a+NxTLFZlO8AY*SfBEqbW7MBxpHIw>*}9?JAn7=h9w&r0+&3#bBAf zT7b?{bptywbzl7^%M)o!-V!rH9adVb=8v?`1SHjvG@=d~1_x7Vt5Wq1PE|@mjyP^f z(A;Ae9QR-~;YXV&X;!Mk$zxtKxrEw=*jL#;p5KAosl9$|EvXVW;+To8Ofha7F^KdQ`h zwj}7-pLLAJ2G6fQ+-a{%_@+Mf^x+}BbVq-$erPhU2*4V3i_+DfY2GCqY@ zvQ0}k+<)iP)`b4`Xk8X{Qz$txiNY$d9f;=HzOKR+0jpkM*Xe7%z(Uo>ouhm><4c#J z=2oghK1IaA7*J0lQ!5UZs%aO`UiOPc2_dpJdX^6?(X2UcsI3tgE(0RfBS!9R&<|IG z*1dLWRePj?xvCzc9uH_dM5VN)rl3)`)KF{n$*JU!HTlC>gSfU)apZU8A+?}izTJ9X z%HsigZC7h$YKlMhf3L2iH}0;3)?Zo;6zPF(_s2LN#8~~bSDb=izEl?b!_)yf8QAeB D+e$-u literal 0 HcmV?d00001 diff --git a/com.ibm.streamsx.messaging/com.ibm.streamsx.messaging.kafka/KafkaProducer/.KafkaProducer.xml~ b/com.ibm.streamsx.messaging/com.ibm.streamsx.messaging.kafka/KafkaProducer/.KafkaProducer.xml~ new file mode 100644 index 0000000..15d3235 --- /dev/null +++ b/com.ibm.streamsx.messaging/com.ibm.streamsx.messaging.kafka/KafkaProducer/.KafkaProducer.xml~ @@ -0,0 +1,83 @@ + + + + + + This operator acts as a Kafka producer sending tuples as mesages to a Kafka broker. The broker is assumed to be already configured and running. The incoming stream can have three attributes: topic, key and message. The message is a required attribute. If the key attribute is not specified, the message is used as the key. A topic can be specified as either an input stream attribute or as a parameter. + +**Behavior in a Consistent Region** +This operator can participate in a consistent region. This operator cannot be placed at the start of a consistent region. + KafkaProducer_16.gif + KafkaProducer_32.gif + + + com.ibm.streamsx.messaging.kafka.KafkaSink + + + + Operator class library + + ../../impl/java/bin + ../../opt/downloaded/* + + + + + + + kafkaProperty + Specify a Kafka property "key=value" form. This will override any property specified in the properties file. + true + rstring + -1 + + + keyAttribute + Name of the attribute for the key. Default is "key" + true + rstring + 1 + + + messageAttribute + Name of the attribute for the message. This attribute is required. Default is "message" + true + rstring + 1 + + + propertiesFile + Properties file containing kafka properties. Properties file is recommended to be stored in the etc directory. If a relative path is specified, the path is relative to the application directory. + true + rstring + 1 + + + topic + Topic to be published to. A topic can also be specified as an input stream attribute. + true + rstring + -1 + + + topicAttribute + Name of the attribute for the topic. Default is "topic" + true + rstring + 1 + + + + + The tuples arriving on this port are expected to contain three attributes "key", "topic" and "message". Out of these "message", is a required attribute. + + NonWindowed + Oblivious + false + 1 + false + + + + + \ No newline at end of file diff --git a/com.ibm.streamsx.messaging/com.ibm.streamsx.messaging.kafka/KafkaProducer/.KafkaProducer_16.gif~ b/com.ibm.streamsx.messaging/com.ibm.streamsx.messaging.kafka/KafkaProducer/.KafkaProducer_16.gif~ new file mode 100644 index 0000000000000000000000000000000000000000..f6ad616cbc32832ce0b66e19f1ff2d112efe6bb3 GIT binary patch literal 618 zcmZ?wbhEHb6krfwc$UdfJYn~WT~`iYeER3r`c-?bUw!mp+0H8!Q}#W-vhm8+#``b7 z-hA@u>>{6=n-WgkczN^br;B&rF5Y&jeeQt=ufE-R{-t;2xyQ#E8)h9mJl%I=xya;o z7i(r5yt}8UW!{nYc?Xv7zFIT=K+Bv1OLty5c&qmjtUfkr z-NjE|e@#`Bw6O|~~GEZcE;>Y6jnvkzQ(@V;&Cf$u+lZ><)2 z`|;cLM;~tNov>`zmDPK$HOxG)aK*~HS%(|v9o@HW>F3w?wk+u^o3yuM!O`^xZd|zY zcH-I#OD1Goe(?VOkts{IUuvCupnvuG3wPc$%{|<`;>@Y5PZw^vSU=;yw;#XeuiLfz z)V)Xh3l?p=G;PhYE@=$IM+7+NLpGN0|UeV z|Nns`g+TEq3nK$VCW8*h98jDvupe*8Y--XKPiyaRk8I9N)Yg-ZVVTUrW)f!6?CTs5 zEX%ZjiOtV6gkyP5h5GV^g}HE#x0E-q_% zMmYmXUf$TK6di_mVPS6D!=ao{Bs3hf7}P~XxZ@bj?V4>|r9^{Rf3hkWD=KF+bu@4) zvGC3^=9Af@@!6UEfTLi~0?UGD2N(rZc$P3$d^jW7%)Y*cC9~kd%-&2JBPLA`Rfnn4 Xk#atrpPslos#?#hnK>y%fx#L8rIH=8 literal 0 HcmV?d00001 diff --git a/com.ibm.streamsx.messaging/com.ibm.streamsx.messaging.kafka/KafkaProducer/.KafkaProducer_32.gif~ b/com.ibm.streamsx.messaging/com.ibm.streamsx.messaging.kafka/KafkaProducer/.KafkaProducer_32.gif~ new file mode 100644 index 0000000000000000000000000000000000000000..56f392ac105f2df9cfed5de2b4d35b32f4b03814 GIT binary patch literal 2602 zcmds2X2bPxc6H>E(=(CWt28l7#``l4{4@7eg3%rZ%0 zO|C*W_VP_aep{g!7^>MPJGrJL=G)@KfL6Bac7?X`VZZ8rdQnZ6th!UK(rZeNzw1vb zI%?8Ue7q!uU)tE4T%>Nx`7!@$NTytUDp{3P_NcPCe?%RdbK|16EV4>HR9K^LY^d!Y z9#^ytsFG--<8MmqyRXaCkMAEpld95WZ;?l>d7_Mz)plLTXsV9I4s{eitK${bJ}GPJ zEpP6ZB-E&H``@Y5l`9pwmAZn1z>DsNk|V$`g}d+7>z{Sx6dhSudJM=a*X1fZJNw6~ zTKe)tz(|V}5vwUao-TjX**BVbPjk1iSDIW`Sy>4Hz|_=T3?H4Y9cZ6>XV1|b>i7eDc5713a z5_9+*SVRZKoc&xON$ifIvv>@cgp3W#7!){dA_{d!c^Ni91E&v90E2i(JerBcL7*$q z5yCqYA!t2_!$M9N2#diH(GWy(b|pbBNa;*B6v8H8?IHP-DId6ynL8>(B;u1Wm_vsS zIUaIy{7!QwDD92&7e3&Xe~x){wB+Rk_&!$PKj%@?tGT+rZ=&fpyoxucLs z|099J_h;`75eYv!=L#LCML9Bg9E_OG$3Tu)j9}Wvpl=36rG6g4;e50fiu~dK2LDO_ z!hkS7jPZwsyaNIzjF{X0h3fM}2Ez{;NL0EY6h>~5LfGXcE)?-ys>yZ(TR-pcA4?qbN&a>=lOeGq0*1UwEneexvsCjy|uEEr1iM;9396!C{bqFouNtO9IX;Az5u0ek>bYq}+_A{yjPI$NP70-@N|)w^uL6e;phB*Nc(o!_S5W2cGu#_4ah@pLBIT z)@dL8qUlh#x3#u3KYY;C*if&ktF5W7QYtDd%I}xmliw{ZDK3%~7UbvU-npHVeJd+7 zBmL%$w4YN`lCLKvCS1FE<#K#n?4^q_(HGAD^ITNq*)yl5r%s-TkQ_gD^vL1xF!7;- z2Sh@_KlnT@XMgBEcF5j6Fbg@Q;N81^`umSR>L9smhaP3)}tDq?fDD zSlD=^#m4j`t?*!2YwHX8DV|nxaJ{8(@H|UGpjA*q;Xq@o!=k0V!qAK7b`}O%B`9Q> z+X=413&bE++|3{tezHW9dP|)dajB71@I3W~Ltx*7DxEP^nipKX=keN{+T~w_sP|== z`nFnbCs1i8JWF$bqPX_e5oV?3?RFrarA2m-10y#YFJ8CH?be*;#opx2SGURw6W1Ku zWo@L8%Z41?_!Zg?O2n`_%aZ^6OH|XzNBPV-5fM(O7C`d_Bl_}G%?cde*zB8aUFMvf zHv8@}6!+!A4Y{**tGBr;UOe8~PSZA*AC7k`blyE>wY_5Ea%*psUI-(Uq^xt+JOa4r^d|o6pf0opv}Aqw%&~z{=`9oe=c4 z!7DGH5xI7)y)`l^vnuY7^f}dXW zTjpSQ(Q>ZcrUia4A{21}fkb#M?O^*;^!$ewZC4gby&5mG0t3#Wi@R_q&?~L&PM~o~ z*Y?B;!h_E(GSlIz&;oAQU?R%=sh3enjKXMkiPl?^WGcdX&bHI~1M^I;U$dWWay~{0 zMCsdyuh}k%QKDBC>5MW=CL@g#jeiDB(oDrWmgS_|nZ-qzSuK%ddmbHppM|#PFB{2> zS}NIVbU673hlEl-O=8}BsnyG + + + + + This operator acts as a Kafka producer sending tuples as mesages to a Kafka broker. The broker is assumed to be already configured and running. The incoming stream can have three attributes: topic, key and message. The message is a required attribute. If the key attribute is not specified, the message is used as the key. A topic can be specified as either an input stream attribute or as a parameter. + +**Behavior in a Consistent Region** +This operator can participate in a consistent region. This operator cannot be placed at the start of a consistent region. + KafkaProducer_16.gif + KafkaProducer_32.gif + + + com.ibm.streamsx.messaging.kafka.KafkaSink + + + + Operator class library + + ../../impl/java/bin + ../../opt/downloaded/* + + + + + + + kafkaProperty + Specify a Kafka property "key=value" form. This will override any property specified in the properties file. + true + rstring + -1 + + + keyAttribute + Name of the attribute for the key. Default is "key" + true + rstring + 1 + + + messageAttribute + Name of the attribute for the message. This attribute is required. Default is "message" + true + rstring + 1 + + + propertiesFile + Properties file containing kafka properties. Properties file is recommended to be stored in the etc directory. If a relative path is specified, the path is relative to the application directory. + true + rstring + 1 + + + topic + Topic to be published to. A topic can also be specified as an input stream attribute. + true + rstring + -1 + + + topicAttribute + Name of the attribute for the topic. Default is "topic" + true + rstring + 1 + + + + + The tuples arriving on this port are expected to contain three attributes "key", "topic" and "message". Out of these "message", is a required attribute. + + NonWindowed + Oblivious + false + 1 + false + + + + + \ No newline at end of file diff --git a/com.ibm.streamsx.messaging/com.ibm.streamsx.messaging.kafka/KafkaProducer/KafkaProducer_16.gif b/com.ibm.streamsx.messaging/com.ibm.streamsx.messaging.kafka/KafkaProducer/KafkaProducer_16.gif new file mode 100644 index 0000000000000000000000000000000000000000..f6ad616cbc32832ce0b66e19f1ff2d112efe6bb3 GIT binary patch literal 618 zcmZ?wbhEHb6krfwc$UdfJYn~WT~`iYeER3r`c-?bUw!mp+0H8!Q}#W-vhm8+#``b7 z-hA@u>>{6=n-WgkczN^br;B&rF5Y&jeeQt=ufE-R{-t;2xyQ#E8)h9mJl%I=xya;o z7i(r5yt}8UW!{nYc?Xv7zFIT=K+Bv1OLty5c&qmjtUfkr z-NjE|e@#`Bw6O|~~GEZcE;>Y6jnvkzQ(@V;&Cf$u+lZ><)2 z`|;cLM;~tNov>`zmDPK$HOxG)aK*~HS%(|v9o@HW>F3w?wk+u^o3yuM!O`^xZd|zY zcH-I#OD1Goe(?VOkts{IUuvCupnvuG3wPc$%{|<`;>@Y5PZw^vSU=;yw;#XeuiLfz z)V)Xh3l?p=G;PhYE@=$IM+7+NLpGN0|UeV z|Nns`g+TEq3nK$VCW8*h98jDvupe*8Y--XKPiyaRk8I9N)Yg-ZVVTUrW)f!6?CTs5 zEX%ZjiOtV6gkyP5h5GV^g}HE#x0E-q_% zMmYmXUf$TK6di_mVPS6D!=ao{Bs3hf7}P~XxZ@bj?V4>|r9^{Rf3hkWD=KF+bu@4) zvGC3^=9Af@@!6UEfTLi~0?UGD2N(rZc$P3$d^jW7%)Y*cC9~kd%-&2JBPLA`Rfnn4 Xk#atrpPslos#?#hnK>y%fx#L8rIH=8 literal 0 HcmV?d00001 diff --git a/com.ibm.streamsx.messaging/com.ibm.streamsx.messaging.kafka/KafkaProducer/KafkaProducer_32.gif b/com.ibm.streamsx.messaging/com.ibm.streamsx.messaging.kafka/KafkaProducer/KafkaProducer_32.gif new file mode 100644 index 0000000000000000000000000000000000000000..56f392ac105f2df9cfed5de2b4d35b32f4b03814 GIT binary patch literal 2602 zcmds2X2bPxc6H>E(=(CWt28l7#``l4{4@7eg3%rZ%0 zO|C*W_VP_aep{g!7^>MPJGrJL=G)@KfL6Bac7?X`VZZ8rdQnZ6th!UK(rZeNzw1vb zI%?8Ue7q!uU)tE4T%>Nx`7!@$NTytUDp{3P_NcPCe?%RdbK|16EV4>HR9K^LY^d!Y z9#^ytsFG--<8MmqyRXaCkMAEpld95WZ;?l>d7_Mz)plLTXsV9I4s{eitK${bJ}GPJ zEpP6ZB-E&H``@Y5l`9pwmAZn1z>DsNk|V$`g}d+7>z{Sx6dhSudJM=a*X1fZJNw6~ zTKe)tz(|V}5vwUao-TjX**BVbPjk1iSDIW`Sy>4Hz|_=T3?H4Y9cZ6>XV1|b>i7eDc5713a z5_9+*SVRZKoc&xON$ifIvv>@cgp3W#7!){dA_{d!c^Ni91E&v90E2i(JerBcL7*$q z5yCqYA!t2_!$M9N2#diH(GWy(b|pbBNa;*B6v8H8?IHP-DId6ynL8>(B;u1Wm_vsS zIUaIy{7!QwDD92&7e3&Xe~x){wB+Rk_&!$PKj%@?tGT+rZ=&fpyoxucLs z|099J_h;`75eYv!=L#LCML9Bg9E_OG$3Tu)j9}Wvpl=36rG6g4;e50fiu~dK2LDO_ z!hkS7jPZwsyaNIzjF{X0h3fM}2Ez{;NL0EY6h>~5LfGXcE)?-ys>yZ(TR-pcA4?qbN&a>=lOeGq0*1UwEneexvsCjy|uEEr1iM;9396!C{bqFouNtO9IX;Az5u0ek>bYq}+_A{yjPI$NP70-@N|)w^uL6e;phB*Nc(o!_S5W2cGu#_4ah@pLBIT z)@dL8qUlh#x3#u3KYY;C*if&ktF5W7QYtDd%I}xmliw{ZDK3%~7UbvU-npHVeJd+7 zBmL%$w4YN`lCLKvCS1FE<#K#n?4^q_(HGAD^ITNq*)yl5r%s-TkQ_gD^vL1xF!7;- z2Sh@_KlnT@XMgBEcF5j6Fbg@Q;N81^`umSR>L9smhaP3)}tDq?fDD zSlD=^#m4j`t?*!2YwHX8DV|nxaJ{8(@H|UGpjA*q;Xq@o!=k0V!qAK7b`}O%B`9Q> z+X=413&bE++|3{tezHW9dP|)dajB71@I3W~Ltx*7DxEP^nipKX=keN{+T~w_sP|== z`nFnbCs1i8JWF$bqPX_e5oV?3?RFrarA2m-10y#YFJ8CH?be*;#opx2SGURw6W1Ku zWo@L8%Z41?_!Zg?O2n`_%aZ^6OH|XzNBPV-5fM(O7C`d_Bl_}G%?cde*zB8aUFMvf zHv8@}6!+!A4Y{**tGBr;UOe8~PSZA*AC7k`blyE>wY_5Ea%*psUI-(Uq^xt+JOa4r^d|o6pf0opv}Aqw%&~z{=`9oe=c4 z!7DGH5xI7)y)`l^vnuY7^f}dXW zTjpSQ(Q>ZcrUia4A{21}fkb#M?O^*;^!$ewZC4gby&5mG0t3#Wi@R_q&?~L&PM~o~ z*Y?B;!h_E(GSlIz&;oAQU?R%=sh3enjKXMkiPl?^WGcdX&bHI~1M^I;U$dWWay~{0 zMCsdyuh}k%QKDBC>5MW=CL@g#jeiDB(oDrWmgS_|nZ-qzSuK%ddmbHppM|#PFB{2> zS}NIVbU673hlEl-O=8}BsnyG + + + + + Java Operator KafkaSource2 + + + com.ibm.streamsx.messaging.kafka.KafkaSource2 + + + + Operator class library + + ../../impl/java/bin + ../../opt/downloaded/* + + + + + + + kafkaProperty + Specify a Kafka property "key=value" form. This will override any property specified in the properties file. + true + rstring + -1 + + + partition + Partition to subscribe to. + false + int32 + 1 + + + propertiesFile + Properties file containing kafka properties. Properties file is recommended to be stored in the etc directory. If a relative path is specified, the path is relative to the application directory. + true + rstring + 1 + + + topic + Topic to be subscribed to. + false + rstring + 1 + + + triggerCount + Number of messages between checkpointing for consistent region. This is only relevant to operator driven checkpointing. + true + int32 + 1 + + + + + + Port that produces tuples + Generating + 1 + false + + + Optional output ports + Generating + + + + \ No newline at end of file From 26b527c88980fba37b8b7d3ee21ee2481552a675 Mon Sep 17 00:00:00 2001 From: Alex-Cook4 Date: Fri, 10 Jul 2015 09:31:59 -0500 Subject: [PATCH 04/14] Merged KafkaSource2 and KafkaSource We will choose which one to use based on user inputs --- com.ibm.streamsx.messaging/.gitignore | 1 + .../.toolkit.xml~ | 130 ----- .../KafkaConsumer/.KafkaConsumer.xml~ | 80 --- .../KafkaConsumer/.KafkaConsumer_16.gif~ | Bin 617 -> 0 bytes .../KafkaConsumer/.KafkaConsumer_32.gif~ | Bin 2617 -> 0 bytes .../KafkaConsumer/KafkaConsumer.xml | 80 --- .../KafkaConsumer/KafkaConsumer_16.gif | Bin 617 -> 0 bytes .../KafkaConsumer/KafkaConsumer_32.gif | Bin 2617 -> 0 bytes .../KafkaProducer/.KafkaProducer.xml~ | 83 --- .../KafkaProducer/.KafkaProducer_16.gif~ | Bin 618 -> 0 bytes .../KafkaProducer/.KafkaProducer_32.gif~ | Bin 2602 -> 0 bytes .../KafkaProducer/KafkaProducer.xml | 83 --- .../KafkaProducer/KafkaProducer_16.gif | Bin 618 -> 0 bytes .../KafkaProducer/KafkaProducer_32.gif | Bin 2602 -> 0 bytes .../KafkaSource2/KafkaSource2.xml | 72 --- .../messaging/kafka/AttributeHelper.java | 3 +- .../messaging/kafka/KafkaBaseOper.java | 17 +- .../streamsx/messaging/kafka/KafkaSource.java | 123 +++- .../messaging/kafka/KafkaSource2.java | 533 ------------------ .../messaging/kafka/SimpleConsumerClient.java | 492 ++++++++++++++++ 20 files changed, 619 insertions(+), 1078 deletions(-) delete mode 100644 com.ibm.streamsx.messaging/com.ibm.streamsx.messaging.kafka/.toolkit.xml~ delete mode 100644 com.ibm.streamsx.messaging/com.ibm.streamsx.messaging.kafka/KafkaConsumer/.KafkaConsumer.xml~ delete mode 100644 com.ibm.streamsx.messaging/com.ibm.streamsx.messaging.kafka/KafkaConsumer/.KafkaConsumer_16.gif~ delete mode 100644 com.ibm.streamsx.messaging/com.ibm.streamsx.messaging.kafka/KafkaConsumer/.KafkaConsumer_32.gif~ delete mode 100644 com.ibm.streamsx.messaging/com.ibm.streamsx.messaging.kafka/KafkaConsumer/KafkaConsumer.xml delete mode 100644 com.ibm.streamsx.messaging/com.ibm.streamsx.messaging.kafka/KafkaConsumer/KafkaConsumer_16.gif delete mode 100644 com.ibm.streamsx.messaging/com.ibm.streamsx.messaging.kafka/KafkaConsumer/KafkaConsumer_32.gif delete mode 100644 com.ibm.streamsx.messaging/com.ibm.streamsx.messaging.kafka/KafkaProducer/.KafkaProducer.xml~ delete mode 100644 com.ibm.streamsx.messaging/com.ibm.streamsx.messaging.kafka/KafkaProducer/.KafkaProducer_16.gif~ delete mode 100644 com.ibm.streamsx.messaging/com.ibm.streamsx.messaging.kafka/KafkaProducer/.KafkaProducer_32.gif~ delete mode 100644 com.ibm.streamsx.messaging/com.ibm.streamsx.messaging.kafka/KafkaProducer/KafkaProducer.xml delete mode 100644 com.ibm.streamsx.messaging/com.ibm.streamsx.messaging.kafka/KafkaProducer/KafkaProducer_16.gif delete mode 100644 com.ibm.streamsx.messaging/com.ibm.streamsx.messaging.kafka/KafkaProducer/KafkaProducer_32.gif delete mode 100644 com.ibm.streamsx.messaging/com.ibm.streamsx.messaging.kafka/KafkaSource2/KafkaSource2.xml delete mode 100644 com.ibm.streamsx.messaging/impl/java/src/com/ibm/streamsx/messaging/kafka/KafkaSource2.java create mode 100644 com.ibm.streamsx.messaging/impl/java/src/com/ibm/streamsx/messaging/kafka/SimpleConsumerClient.java diff --git a/com.ibm.streamsx.messaging/.gitignore b/com.ibm.streamsx.messaging/.gitignore index df6330a..baaee37 100644 --- a/com.ibm.streamsx.messaging/.gitignore +++ b/com.ibm.streamsx.messaging/.gitignore @@ -1,5 +1,6 @@ /output impl/lib +com.ibm.streamsx.messaging.kafka opt/downloaded /target /.externalToolBuilders diff --git a/com.ibm.streamsx.messaging/com.ibm.streamsx.messaging.kafka/.toolkit.xml~ b/com.ibm.streamsx.messaging/com.ibm.streamsx.messaging.kafka/.toolkit.xml~ deleted file mode 100644 index c3d1a16..0000000 --- a/com.ibm.streamsx.messaging/com.ibm.streamsx.messaging.kafka/.toolkit.xml~ +++ /dev/null @@ -1,130 +0,0 @@ - - - - - Toolkit Description - - - - - - - - - - - - This operator acts as a Kafka producer sending tuples as mesages to a Kafka broker. The broker is assumed to be already configured and running. The incoming stream can have three attributes: topic, key and message. The message is a required attribute. If the key attribute is not specified, the message is used as the key. A topic can be specified as either an input stream attribute or as a parameter. - -**Behavior in a Consistent Region** -This operator can participate in a consistent region. This operator cannot be placed at the start of a consistent region. - - - - - - Command line options passed to the Java Virtual Machine at startup - - - Specify a Kafka property "key=value" form. This will override any property specified in the properties file. - - - Name of the attribute for the key. Default is "key" - - - Name of the attribute for the message. This attribute is required. Default is "message" - - - Properties file containing kafka properties. Properties file is recommended to be stored in the etc directory. If a relative path is specified, the path is relative to the application directory. - - - Topic to be published to. A topic can also be specified as an input stream attribute. - - - Name of the attribute for the topic. Default is "topic" - - - The tuples arriving on this port are expected to contain three attributes "key", "topic" and "message". Out of these "message", is a required attribute. - NonWindowed - - - - This operator acts as a Kafka consumer receiving messages for one or more topics. Note that there may be multiple threads receiving messages depending on the configuration specified. Ordering of messages is not guaranteed. - -**Behavior in a Consistent Region** -This operator cannot be used inside a consistent region. - - - - - - Command line options passed to the Java Virtual Machine at startup - - - Specify a Kafka property "key=value" form. This will override any property specified in the properties file. - - - Name of the attribute for the key. Default is "key" - - - Name of the attribute for the message. This attribute is required. Default is "message" - - - Properties file containing kafka properties. Properties file is recommended to be stored in the etc directory. If a relative path is specified, the path is relative to the application directory. - - - Number of threads per topic. Default is 1. - - - Topic to be subscribed to. - - - Messages received from Kafka are sent on this output port. - - - - Java Operator NewKafkaConsumer - - Command line options passed to the Java Virtual Machine at startup - - - Specify a Kafka property "key=value" form. This will override any property specified in the properties file. - - - Name of the attribute for the key. Default is "key" - - - Name of the attribute for the message. This attribute is required. Default is "message" - - - Properties file containing kafka properties. Properties file is recommended to be stored in the etc directory. If a relative path is specified, the path is relative to the application directory. - - - Number of threads per topic. Default is 1. - - - Topic to be subscribed to. - - - Port that produces tuples - - - Optional output ports - - - - - - - - - - - - - - - - - - diff --git a/com.ibm.streamsx.messaging/com.ibm.streamsx.messaging.kafka/KafkaConsumer/.KafkaConsumer.xml~ b/com.ibm.streamsx.messaging/com.ibm.streamsx.messaging.kafka/KafkaConsumer/.KafkaConsumer.xml~ deleted file mode 100644 index 7be8c51..0000000 --- a/com.ibm.streamsx.messaging/com.ibm.streamsx.messaging.kafka/KafkaConsumer/.KafkaConsumer.xml~ +++ /dev/null @@ -1,80 +0,0 @@ - - - - - - This operator acts as a Kafka consumer receiving messages for one or more topics. Note that there may be multiple threads receiving messages depending on the configuration specified. Ordering of messages is not guaranteed. - -**Behavior in a Consistent Region** -This operator cannot be used inside a consistent region. - KafkaConsumer_16.gif - KafkaConsumer_32.gif - - - com.ibm.streamsx.messaging.kafka.KafkaSource - - - - Operator class library - - ../../impl/java/bin - ../../opt/downloaded/* - - - - - - - kafkaProperty - Specify a Kafka property "key=value" form. This will override any property specified in the properties file. - true - rstring - -1 - - - keyAttribute - Name of the attribute for the key. Default is "key" - true - rstring - 1 - - - messageAttribute - Name of the attribute for the message. This attribute is required. Default is "message" - true - rstring - 1 - - - propertiesFile - Properties file containing kafka properties. Properties file is recommended to be stored in the etc directory. If a relative path is specified, the path is relative to the application directory. - true - rstring - 1 - - - threadsPerTopic - Number of threads per topic. Default is 1. - true - int32 - 1 - - - topic - Topic to be subscribed to. - false - rstring - -1 - - - - - - Messages received from Kafka are sent on this output port. - Free - 1 - false - - - - \ No newline at end of file diff --git a/com.ibm.streamsx.messaging/com.ibm.streamsx.messaging.kafka/KafkaConsumer/.KafkaConsumer_16.gif~ b/com.ibm.streamsx.messaging/com.ibm.streamsx.messaging.kafka/KafkaConsumer/.KafkaConsumer_16.gif~ deleted file mode 100644 index 12b21132b368dde4bfeaedc8b710c36907388043..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 617 zcmZ?wbhEHb6krfwc$UF%_~O%5d#?X^wZ3A?zDesYUVZeTVdjAoH(uU<`Ss@0PiGhT zT)g}C&hsxfHzn*ie)qwvZw<2!F5Y%&YqiL-9hd889(Z%__@b?sKEJ+q=H~0=JFjeS zQm>hD@YC0yU%&s_cl!SBR^_Vc2d+QASC`wOP8Jv(j9+39P} zb}TqrIc494)fp}Gj!ay8VPl14#ngRQ*Tud(UB9thq-pNq8+#{AS#|pCt=9+6KKSzH z&bJRwA0KO6w)0Bw%5$stTq~Klr+dYjU%&sBP1<{Kg4Lz_@7m@ax%2ql@9!TQ=N-Lp z=WXZ0W3O-R-gx--`h&OjEh*c!ZuZP|=PurT_wf0LscX)(%|B8+VRu?u8Uq8v|NsAi zB;`QyCkrD3Lk5Em$S6>pFt8tM$lw!5X>GHT%joKoNtL!>oXBXIY#ONF?U5Q_%P@~2 zN+~u;H@ZtQ)h;!Xo7>$)J6_91-ZLX1l|@B|lhc?@h>OeHAUND@r$Pt^hXk7*(?upn zBlDdOiv0YEY|gyAPG(YGVmtjcd3cmX3^n4^>_wYWeO*|;vxW-Cgt@ABrJqXBEL`wl zrg7Sgk{ugf_^@a)EAd%uKHM|emEod-N%AuVClgs)xtJ6;930tU2z0s^81 zMX`voiGTqWBy6%Jf`qkUCm=hrqk^Z0#?R?ugJVhpx6$-`GSXpdrY*$wo1VM@f@<3JK znKNg2JRS&wqFYkGC|O91EIGgBVx;ux)2IH?(t!IfgJWb-PZU?~$-2v9b1Qr3bb77W zG2njL&E)c=yrzuOjzEmDrhyOB=2gb@%W3E)i(^K-Beeafjn4ZWj>xfFPlXwF|W!%D=zR<*qrfgqjrsHTs zM2-(I(vp?w1w@Fe+X{R;i})8JW$}`R9AC}GEMIB;Ktr0X*KJwcZQo39pgiVCllVY> zWzX!)ui}E1>`2?^<(=6TJ=qt5z!-T-p>n7(wIIYO^0A_#ap3vSc29*sQ-aYgd; zrofo8;^JZe0OsfCk@^33Y?xnqj7A5k3?T#g?%XZnF_^Bf5Oji_*&GY>yZmA_$Yxof z9SAgt#-qS4Y#VPrZ0CK*p6Tt%B(l)EcY!-aB#}GM9TqY`k-Hm5KoVJ?87!_7OhVSG zZ7dpGP!YOXpep$9Kps#3shLZcleR!;3;);o&)OH*d+}hbEiB-A@R=~8zVRlwgIof(Qhn2^F&1zG&lfSaU0ai3tV5Z71TWINTmnONanbs1%Bom5HS_ zo=9B8?&AoA3=R`sBKO@ zn-iQ&wiD1F@vQ$PaahD5tg7JtQh`F>*j-P<=~qa(vZg9H7q`+9r2yE;4CU$wO=Tbi318|v$76~EL}S5;P&%gbJt zmb{Qji;D^i@}KAB=459{GM{CnKYj8zEmiy|B{}I~;)8_vxY(Hc_oDAc{d_0#_N|DU zH^Reyx_&M6YDjR%aSl{fxzFEC$<<~2gFZ)Vw=@MO(j<%MjhPoO6Y?#qR zW+|{u_3Wzt;{a+NxTLFZlO8AY*SfBEqbW7MBxpHIw>*}9?JAn7=h9w&r0+&3#bBAf zT7b?{bptywbzl7^%M)o!-V!rH9adVb=8v?`1SHjvG@=d~1_x7Vt5Wq1PE|@mjyP^f z(A;Ae9QR-~;YXV&X;!Mk$zxtKxrEw=*jL#;p5KAosl9$|EvXVW;+To8Ofha7F^KdQ`h zwj}7-pLLAJ2G6fQ+-a{%_@+Mf^x+}BbVq-$erPhU2*4V3i_+DfY2GCqY@ zvQ0}k+<)iP)`b4`Xk8X{Qz$txiNY$d9f;=HzOKR+0jpkM*Xe7%z(Uo>ouhm><4c#J z=2oghK1IaA7*J0lQ!5UZs%aO`UiOPc2_dpJdX^6?(X2UcsI3tgE(0RfBS!9R&<|IG z*1dLWRePj?xvCzc9uH_dM5VN)rl3)`)KF{n$*JU!HTlC>gSfU)apZU8A+?}izTJ9X z%HsigZC7h$YKlMhf3L2iH}0;3)?Zo;6zPF(_s2LN#8~~bSDb=izEl?b!_)yf8QAeB D+e$-u diff --git a/com.ibm.streamsx.messaging/com.ibm.streamsx.messaging.kafka/KafkaConsumer/KafkaConsumer.xml b/com.ibm.streamsx.messaging/com.ibm.streamsx.messaging.kafka/KafkaConsumer/KafkaConsumer.xml deleted file mode 100644 index 7d0d1aa..0000000 --- a/com.ibm.streamsx.messaging/com.ibm.streamsx.messaging.kafka/KafkaConsumer/KafkaConsumer.xml +++ /dev/null @@ -1,80 +0,0 @@ - - - - - - This operator acts as a Kafka consumer receiving messages for one or more topics. Note that there may be multiple threads receiving messages depending on the configuration specified. Ordering of messages is not guaranteed. - -**Behavior in a Consistent Region** -This operator cannot be used inside a consistent region. - KafkaConsumer_16.gif - KafkaConsumer_32.gif - - - com.ibm.streamsx.messaging.kafka.KafkaSource - - - - Operator class library - - ../../impl/java/bin - ../../opt/downloaded/* - - - - - - - kafkaProperty - Specify a Kafka property "key=value" form. This will override any property specified in the properties file. - true - rstring - -1 - - - keyAttribute - Name of the attribute for the key. Default is "key" - true - rstring - 1 - - - messageAttribute - Name of the attribute for the message. This attribute is required. Default is "message" - true - rstring - 1 - - - propertiesFile - Properties file containing kafka properties. Properties file is recommended to be stored in the etc directory. If a relative path is specified, the path is relative to the application directory. - true - rstring - 1 - - - threadsPerTopic - Number of threads per topic. Default is 1. - true - int32 - 1 - - - topic - Topic to be subscribed to. - false - rstring - -1 - - - - - - Messages received from Kafka are sent on this output port. - Free - 1 - false - - - - \ No newline at end of file diff --git a/com.ibm.streamsx.messaging/com.ibm.streamsx.messaging.kafka/KafkaConsumer/KafkaConsumer_16.gif b/com.ibm.streamsx.messaging/com.ibm.streamsx.messaging.kafka/KafkaConsumer/KafkaConsumer_16.gif deleted file mode 100644 index 12b21132b368dde4bfeaedc8b710c36907388043..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 617 zcmZ?wbhEHb6krfwc$UF%_~O%5d#?X^wZ3A?zDesYUVZeTVdjAoH(uU<`Ss@0PiGhT zT)g}C&hsxfHzn*ie)qwvZw<2!F5Y%&YqiL-9hd889(Z%__@b?sKEJ+q=H~0=JFjeS zQm>hD@YC0yU%&s_cl!SBR^_Vc2d+QASC`wOP8Jv(j9+39P} zb}TqrIc494)fp}Gj!ay8VPl14#ngRQ*Tud(UB9thq-pNq8+#{AS#|pCt=9+6KKSzH z&bJRwA0KO6w)0Bw%5$stTq~Klr+dYjU%&sBP1<{Kg4Lz_@7m@ax%2ql@9!TQ=N-Lp z=WXZ0W3O-R-gx--`h&OjEh*c!ZuZP|=PurT_wf0LscX)(%|B8+VRu?u8Uq8v|NsAi zB;`QyCkrD3Lk5Em$S6>pFt8tM$lw!5X>GHT%joKoNtL!>oXBXIY#ONF?U5Q_%P@~2 zN+~u;H@ZtQ)h;!Xo7>$)J6_91-ZLX1l|@B|lhc?@h>OeHAUND@r$Pt^hXk7*(?upn zBlDdOiv0YEY|gyAPG(YGVmtjcd3cmX3^n4^>_wYWeO*|;vxW-Cgt@ABrJqXBEL`wl zrg7Sgk{ugf_^@a)EAd%uKHM|emEod-N%AuVClgs)xtJ6;930tU2z0s^81 zMX`voiGTqWBy6%Jf`qkUCm=hrqk^Z0#?R?ugJVhpx6$-`GSXpdrY*$wo1VM@f@<3JK znKNg2JRS&wqFYkGC|O91EIGgBVx;ux)2IH?(t!IfgJWb-PZU?~$-2v9b1Qr3bb77W zG2njL&E)c=yrzuOjzEmDrhyOB=2gb@%W3E)i(^K-Beeafjn4ZWj>xfFPlXwF|W!%D=zR<*qrfgqjrsHTs zM2-(I(vp?w1w@Fe+X{R;i})8JW$}`R9AC}GEMIB;Ktr0X*KJwcZQo39pgiVCllVY> zWzX!)ui}E1>`2?^<(=6TJ=qt5z!-T-p>n7(wIIYO^0A_#ap3vSc29*sQ-aYgd; zrofo8;^JZe0OsfCk@^33Y?xnqj7A5k3?T#g?%XZnF_^Bf5Oji_*&GY>yZmA_$Yxof z9SAgt#-qS4Y#VPrZ0CK*p6Tt%B(l)EcY!-aB#}GM9TqY`k-Hm5KoVJ?87!_7OhVSG zZ7dpGP!YOXpep$9Kps#3shLZcleR!;3;);o&)OH*d+}hbEiB-A@R=~8zVRlwgIof(Qhn2^F&1zG&lfSaU0ai3tV5Z71TWINTmnONanbs1%Bom5HS_ zo=9B8?&AoA3=R`sBKO@ zn-iQ&wiD1F@vQ$PaahD5tg7JtQh`F>*j-P<=~qa(vZg9H7q`+9r2yE;4CU$wO=Tbi318|v$76~EL}S5;P&%gbJt zmb{Qji;D^i@}KAB=459{GM{CnKYj8zEmiy|B{}I~;)8_vxY(Hc_oDAc{d_0#_N|DU zH^Reyx_&M6YDjR%aSl{fxzFEC$<<~2gFZ)Vw=@MO(j<%MjhPoO6Y?#qR zW+|{u_3Wzt;{a+NxTLFZlO8AY*SfBEqbW7MBxpHIw>*}9?JAn7=h9w&r0+&3#bBAf zT7b?{bptywbzl7^%M)o!-V!rH9adVb=8v?`1SHjvG@=d~1_x7Vt5Wq1PE|@mjyP^f z(A;Ae9QR-~;YXV&X;!Mk$zxtKxrEw=*jL#;p5KAosl9$|EvXVW;+To8Ofha7F^KdQ`h zwj}7-pLLAJ2G6fQ+-a{%_@+Mf^x+}BbVq-$erPhU2*4V3i_+DfY2GCqY@ zvQ0}k+<)iP)`b4`Xk8X{Qz$txiNY$d9f;=HzOKR+0jpkM*Xe7%z(Uo>ouhm><4c#J z=2oghK1IaA7*J0lQ!5UZs%aO`UiOPc2_dpJdX^6?(X2UcsI3tgE(0RfBS!9R&<|IG z*1dLWRePj?xvCzc9uH_dM5VN)rl3)`)KF{n$*JU!HTlC>gSfU)apZU8A+?}izTJ9X z%HsigZC7h$YKlMhf3L2iH}0;3)?Zo;6zPF(_s2LN#8~~bSDb=izEl?b!_)yf8QAeB D+e$-u diff --git a/com.ibm.streamsx.messaging/com.ibm.streamsx.messaging.kafka/KafkaProducer/.KafkaProducer.xml~ b/com.ibm.streamsx.messaging/com.ibm.streamsx.messaging.kafka/KafkaProducer/.KafkaProducer.xml~ deleted file mode 100644 index 15d3235..0000000 --- a/com.ibm.streamsx.messaging/com.ibm.streamsx.messaging.kafka/KafkaProducer/.KafkaProducer.xml~ +++ /dev/null @@ -1,83 +0,0 @@ - - - - - - This operator acts as a Kafka producer sending tuples as mesages to a Kafka broker. The broker is assumed to be already configured and running. The incoming stream can have three attributes: topic, key and message. The message is a required attribute. If the key attribute is not specified, the message is used as the key. A topic can be specified as either an input stream attribute or as a parameter. - -**Behavior in a Consistent Region** -This operator can participate in a consistent region. This operator cannot be placed at the start of a consistent region. - KafkaProducer_16.gif - KafkaProducer_32.gif - - - com.ibm.streamsx.messaging.kafka.KafkaSink - - - - Operator class library - - ../../impl/java/bin - ../../opt/downloaded/* - - - - - - - kafkaProperty - Specify a Kafka property "key=value" form. This will override any property specified in the properties file. - true - rstring - -1 - - - keyAttribute - Name of the attribute for the key. Default is "key" - true - rstring - 1 - - - messageAttribute - Name of the attribute for the message. This attribute is required. Default is "message" - true - rstring - 1 - - - propertiesFile - Properties file containing kafka properties. Properties file is recommended to be stored in the etc directory. If a relative path is specified, the path is relative to the application directory. - true - rstring - 1 - - - topic - Topic to be published to. A topic can also be specified as an input stream attribute. - true - rstring - -1 - - - topicAttribute - Name of the attribute for the topic. Default is "topic" - true - rstring - 1 - - - - - The tuples arriving on this port are expected to contain three attributes "key", "topic" and "message". Out of these "message", is a required attribute. - - NonWindowed - Oblivious - false - 1 - false - - - - - \ No newline at end of file diff --git a/com.ibm.streamsx.messaging/com.ibm.streamsx.messaging.kafka/KafkaProducer/.KafkaProducer_16.gif~ b/com.ibm.streamsx.messaging/com.ibm.streamsx.messaging.kafka/KafkaProducer/.KafkaProducer_16.gif~ deleted file mode 100644 index f6ad616cbc32832ce0b66e19f1ff2d112efe6bb3..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 618 zcmZ?wbhEHb6krfwc$UdfJYn~WT~`iYeER3r`c-?bUw!mp+0H8!Q}#W-vhm8+#``b7 z-hA@u>>{6=n-WgkczN^br;B&rF5Y&jeeQt=ufE-R{-t;2xyQ#E8)h9mJl%I=xya;o z7i(r5yt}8UW!{nYc?Xv7zFIT=K+Bv1OLty5c&qmjtUfkr z-NjE|e@#`Bw6O|~~GEZcE;>Y6jnvkzQ(@V;&Cf$u+lZ><)2 z`|;cLM;~tNov>`zmDPK$HOxG)aK*~HS%(|v9o@HW>F3w?wk+u^o3yuM!O`^xZd|zY zcH-I#OD1Goe(?VOkts{IUuvCupnvuG3wPc$%{|<`;>@Y5PZw^vSU=;yw;#XeuiLfz z)V)Xh3l?p=G;PhYE@=$IM+7+NLpGN0|UeV z|Nns`g+TEq3nK$VCW8*h98jDvupe*8Y--XKPiyaRk8I9N)Yg-ZVVTUrW)f!6?CTs5 zEX%ZjiOtV6gkyP5h5GV^g}HE#x0E-q_% zMmYmXUf$TK6di_mVPS6D!=ao{Bs3hf7}P~XxZ@bj?V4>|r9^{Rf3hkWD=KF+bu@4) zvGC3^=9Af@@!6UEfTLi~0?UGD2N(rZc$P3$d^jW7%)Y*cC9~kd%-&2JBPLA`Rfnn4 Xk#atrpPslos#?#hnK>y%fx#L8rIH=8 diff --git a/com.ibm.streamsx.messaging/com.ibm.streamsx.messaging.kafka/KafkaProducer/.KafkaProducer_32.gif~ b/com.ibm.streamsx.messaging/com.ibm.streamsx.messaging.kafka/KafkaProducer/.KafkaProducer_32.gif~ deleted file mode 100644 index 56f392ac105f2df9cfed5de2b4d35b32f4b03814..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2602 zcmds2X2bPxc6H>E(=(CWt28l7#``l4{4@7eg3%rZ%0 zO|C*W_VP_aep{g!7^>MPJGrJL=G)@KfL6Bac7?X`VZZ8rdQnZ6th!UK(rZeNzw1vb zI%?8Ue7q!uU)tE4T%>Nx`7!@$NTytUDp{3P_NcPCe?%RdbK|16EV4>HR9K^LY^d!Y z9#^ytsFG--<8MmqyRXaCkMAEpld95WZ;?l>d7_Mz)plLTXsV9I4s{eitK${bJ}GPJ zEpP6ZB-E&H``@Y5l`9pwmAZn1z>DsNk|V$`g}d+7>z{Sx6dhSudJM=a*X1fZJNw6~ zTKe)tz(|V}5vwUao-TjX**BVbPjk1iSDIW`Sy>4Hz|_=T3?H4Y9cZ6>XV1|b>i7eDc5713a z5_9+*SVRZKoc&xON$ifIvv>@cgp3W#7!){dA_{d!c^Ni91E&v90E2i(JerBcL7*$q z5yCqYA!t2_!$M9N2#diH(GWy(b|pbBNa;*B6v8H8?IHP-DId6ynL8>(B;u1Wm_vsS zIUaIy{7!QwDD92&7e3&Xe~x){wB+Rk_&!$PKj%@?tGT+rZ=&fpyoxucLs z|099J_h;`75eYv!=L#LCML9Bg9E_OG$3Tu)j9}Wvpl=36rG6g4;e50fiu~dK2LDO_ z!hkS7jPZwsyaNIzjF{X0h3fM}2Ez{;NL0EY6h>~5LfGXcE)?-ys>yZ(TR-pcA4?qbN&a>=lOeGq0*1UwEneexvsCjy|uEEr1iM;9396!C{bqFouNtO9IX;Az5u0ek>bYq}+_A{yjPI$NP70-@N|)w^uL6e;phB*Nc(o!_S5W2cGu#_4ah@pLBIT z)@dL8qUlh#x3#u3KYY;C*if&ktF5W7QYtDd%I}xmliw{ZDK3%~7UbvU-npHVeJd+7 zBmL%$w4YN`lCLKvCS1FE<#K#n?4^q_(HGAD^ITNq*)yl5r%s-TkQ_gD^vL1xF!7;- z2Sh@_KlnT@XMgBEcF5j6Fbg@Q;N81^`umSR>L9smhaP3)}tDq?fDD zSlD=^#m4j`t?*!2YwHX8DV|nxaJ{8(@H|UGpjA*q;Xq@o!=k0V!qAK7b`}O%B`9Q> z+X=413&bE++|3{tezHW9dP|)dajB71@I3W~Ltx*7DxEP^nipKX=keN{+T~w_sP|== z`nFnbCs1i8JWF$bqPX_e5oV?3?RFrarA2m-10y#YFJ8CH?be*;#opx2SGURw6W1Ku zWo@L8%Z41?_!Zg?O2n`_%aZ^6OH|XzNBPV-5fM(O7C`d_Bl_}G%?cde*zB8aUFMvf zHv8@}6!+!A4Y{**tGBr;UOe8~PSZA*AC7k`blyE>wY_5Ea%*psUI-(Uq^xt+JOa4r^d|o6pf0opv}Aqw%&~z{=`9oe=c4 z!7DGH5xI7)y)`l^vnuY7^f}dXW zTjpSQ(Q>ZcrUia4A{21}fkb#M?O^*;^!$ewZC4gby&5mG0t3#Wi@R_q&?~L&PM~o~ z*Y?B;!h_E(GSlIz&;oAQU?R%=sh3enjKXMkiPl?^WGcdX&bHI~1M^I;U$dWWay~{0 zMCsdyuh}k%QKDBC>5MW=CL@g#jeiDB(oDrWmgS_|nZ-qzSuK%ddmbHppM|#PFB{2> zS}NIVbU673hlEl-O=8}BsnyG - - - - - This operator acts as a Kafka producer sending tuples as mesages to a Kafka broker. The broker is assumed to be already configured and running. The incoming stream can have three attributes: topic, key and message. The message is a required attribute. If the key attribute is not specified, the message is used as the key. A topic can be specified as either an input stream attribute or as a parameter. - -**Behavior in a Consistent Region** -This operator can participate in a consistent region. This operator cannot be placed at the start of a consistent region. - KafkaProducer_16.gif - KafkaProducer_32.gif - - - com.ibm.streamsx.messaging.kafka.KafkaSink - - - - Operator class library - - ../../impl/java/bin - ../../opt/downloaded/* - - - - - - - kafkaProperty - Specify a Kafka property "key=value" form. This will override any property specified in the properties file. - true - rstring - -1 - - - keyAttribute - Name of the attribute for the key. Default is "key" - true - rstring - 1 - - - messageAttribute - Name of the attribute for the message. This attribute is required. Default is "message" - true - rstring - 1 - - - propertiesFile - Properties file containing kafka properties. Properties file is recommended to be stored in the etc directory. If a relative path is specified, the path is relative to the application directory. - true - rstring - 1 - - - topic - Topic to be published to. A topic can also be specified as an input stream attribute. - true - rstring - -1 - - - topicAttribute - Name of the attribute for the topic. Default is "topic" - true - rstring - 1 - - - - - The tuples arriving on this port are expected to contain three attributes "key", "topic" and "message". Out of these "message", is a required attribute. - - NonWindowed - Oblivious - false - 1 - false - - - - - \ No newline at end of file diff --git a/com.ibm.streamsx.messaging/com.ibm.streamsx.messaging.kafka/KafkaProducer/KafkaProducer_16.gif b/com.ibm.streamsx.messaging/com.ibm.streamsx.messaging.kafka/KafkaProducer/KafkaProducer_16.gif deleted file mode 100644 index f6ad616cbc32832ce0b66e19f1ff2d112efe6bb3..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 618 zcmZ?wbhEHb6krfwc$UdfJYn~WT~`iYeER3r`c-?bUw!mp+0H8!Q}#W-vhm8+#``b7 z-hA@u>>{6=n-WgkczN^br;B&rF5Y&jeeQt=ufE-R{-t;2xyQ#E8)h9mJl%I=xya;o z7i(r5yt}8UW!{nYc?Xv7zFIT=K+Bv1OLty5c&qmjtUfkr z-NjE|e@#`Bw6O|~~GEZcE;>Y6jnvkzQ(@V;&Cf$u+lZ><)2 z`|;cLM;~tNov>`zmDPK$HOxG)aK*~HS%(|v9o@HW>F3w?wk+u^o3yuM!O`^xZd|zY zcH-I#OD1Goe(?VOkts{IUuvCupnvuG3wPc$%{|<`;>@Y5PZw^vSU=;yw;#XeuiLfz z)V)Xh3l?p=G;PhYE@=$IM+7+NLpGN0|UeV z|Nns`g+TEq3nK$VCW8*h98jDvupe*8Y--XKPiyaRk8I9N)Yg-ZVVTUrW)f!6?CTs5 zEX%ZjiOtV6gkyP5h5GV^g}HE#x0E-q_% zMmYmXUf$TK6di_mVPS6D!=ao{Bs3hf7}P~XxZ@bj?V4>|r9^{Rf3hkWD=KF+bu@4) zvGC3^=9Af@@!6UEfTLi~0?UGD2N(rZc$P3$d^jW7%)Y*cC9~kd%-&2JBPLA`Rfnn4 Xk#atrpPslos#?#hnK>y%fx#L8rIH=8 diff --git a/com.ibm.streamsx.messaging/com.ibm.streamsx.messaging.kafka/KafkaProducer/KafkaProducer_32.gif b/com.ibm.streamsx.messaging/com.ibm.streamsx.messaging.kafka/KafkaProducer/KafkaProducer_32.gif deleted file mode 100644 index 56f392ac105f2df9cfed5de2b4d35b32f4b03814..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2602 zcmds2X2bPxc6H>E(=(CWt28l7#``l4{4@7eg3%rZ%0 zO|C*W_VP_aep{g!7^>MPJGrJL=G)@KfL6Bac7?X`VZZ8rdQnZ6th!UK(rZeNzw1vb zI%?8Ue7q!uU)tE4T%>Nx`7!@$NTytUDp{3P_NcPCe?%RdbK|16EV4>HR9K^LY^d!Y z9#^ytsFG--<8MmqyRXaCkMAEpld95WZ;?l>d7_Mz)plLTXsV9I4s{eitK${bJ}GPJ zEpP6ZB-E&H``@Y5l`9pwmAZn1z>DsNk|V$`g}d+7>z{Sx6dhSudJM=a*X1fZJNw6~ zTKe)tz(|V}5vwUao-TjX**BVbPjk1iSDIW`Sy>4Hz|_=T3?H4Y9cZ6>XV1|b>i7eDc5713a z5_9+*SVRZKoc&xON$ifIvv>@cgp3W#7!){dA_{d!c^Ni91E&v90E2i(JerBcL7*$q z5yCqYA!t2_!$M9N2#diH(GWy(b|pbBNa;*B6v8H8?IHP-DId6ynL8>(B;u1Wm_vsS zIUaIy{7!QwDD92&7e3&Xe~x){wB+Rk_&!$PKj%@?tGT+rZ=&fpyoxucLs z|099J_h;`75eYv!=L#LCML9Bg9E_OG$3Tu)j9}Wvpl=36rG6g4;e50fiu~dK2LDO_ z!hkS7jPZwsyaNIzjF{X0h3fM}2Ez{;NL0EY6h>~5LfGXcE)?-ys>yZ(TR-pcA4?qbN&a>=lOeGq0*1UwEneexvsCjy|uEEr1iM;9396!C{bqFouNtO9IX;Az5u0ek>bYq}+_A{yjPI$NP70-@N|)w^uL6e;phB*Nc(o!_S5W2cGu#_4ah@pLBIT z)@dL8qUlh#x3#u3KYY;C*if&ktF5W7QYtDd%I}xmliw{ZDK3%~7UbvU-npHVeJd+7 zBmL%$w4YN`lCLKvCS1FE<#K#n?4^q_(HGAD^ITNq*)yl5r%s-TkQ_gD^vL1xF!7;- z2Sh@_KlnT@XMgBEcF5j6Fbg@Q;N81^`umSR>L9smhaP3)}tDq?fDD zSlD=^#m4j`t?*!2YwHX8DV|nxaJ{8(@H|UGpjA*q;Xq@o!=k0V!qAK7b`}O%B`9Q> z+X=413&bE++|3{tezHW9dP|)dajB71@I3W~Ltx*7DxEP^nipKX=keN{+T~w_sP|== z`nFnbCs1i8JWF$bqPX_e5oV?3?RFrarA2m-10y#YFJ8CH?be*;#opx2SGURw6W1Ku zWo@L8%Z41?_!Zg?O2n`_%aZ^6OH|XzNBPV-5fM(O7C`d_Bl_}G%?cde*zB8aUFMvf zHv8@}6!+!A4Y{**tGBr;UOe8~PSZA*AC7k`blyE>wY_5Ea%*psUI-(Uq^xt+JOa4r^d|o6pf0opv}Aqw%&~z{=`9oe=c4 z!7DGH5xI7)y)`l^vnuY7^f}dXW zTjpSQ(Q>ZcrUia4A{21}fkb#M?O^*;^!$ewZC4gby&5mG0t3#Wi@R_q&?~L&PM~o~ z*Y?B;!h_E(GSlIz&;oAQU?R%=sh3enjKXMkiPl?^WGcdX&bHI~1M^I;U$dWWay~{0 zMCsdyuh}k%QKDBC>5MW=CL@g#jeiDB(oDrWmgS_|nZ-qzSuK%ddmbHppM|#PFB{2> zS}NIVbU673hlEl-O=8}BsnyG - - - - - Java Operator KafkaSource2 - - - com.ibm.streamsx.messaging.kafka.KafkaSource2 - - - - Operator class library - - ../../impl/java/bin - ../../opt/downloaded/* - - - - - - - kafkaProperty - Specify a Kafka property "key=value" form. This will override any property specified in the properties file. - true - rstring - -1 - - - partition - Partition to subscribe to. - false - int32 - 1 - - - propertiesFile - Properties file containing kafka properties. Properties file is recommended to be stored in the etc directory. If a relative path is specified, the path is relative to the application directory. - true - rstring - 1 - - - topic - Topic to be subscribed to. - false - rstring - 1 - - - triggerCount - Number of messages between checkpointing for consistent region. This is only relevant to operator driven checkpointing. - true - int32 - 1 - - - - - - Port that produces tuples - Generating - 1 - false - - - Optional output ports - Generating - - - - \ No newline at end of file diff --git a/com.ibm.streamsx.messaging/impl/java/src/com/ibm/streamsx/messaging/kafka/AttributeHelper.java b/com.ibm.streamsx.messaging/impl/java/src/com/ibm/streamsx/messaging/kafka/AttributeHelper.java index ed6531f..664c697 100644 --- a/com.ibm.streamsx.messaging/impl/java/src/com/ibm/streamsx/messaging/kafka/AttributeHelper.java +++ b/com.ibm.streamsx.messaging/impl/java/src/com/ibm/streamsx/messaging/kafka/AttributeHelper.java @@ -81,7 +81,8 @@ void setValue(OutputTuple otup, byte[] value) { otup.setString(name, new String(value, CS)); else otup.setBlob(name, ValueFactory.newBlob(value)); - } + } + String getString(Tuple tuple) throws IOException { if(!isAvailable) return null; if(isString) diff --git a/com.ibm.streamsx.messaging/impl/java/src/com/ibm/streamsx/messaging/kafka/KafkaBaseOper.java b/com.ibm.streamsx.messaging/impl/java/src/com/ibm/streamsx/messaging/kafka/KafkaBaseOper.java index 1bf9b6f..9b790ea 100644 --- a/com.ibm.streamsx.messaging/impl/java/src/com/ibm/streamsx/messaging/kafka/KafkaBaseOper.java +++ b/com.ibm.streamsx.messaging/impl/java/src/com/ibm/streamsx/messaging/kafka/KafkaBaseOper.java @@ -37,6 +37,7 @@ public abstract class KafkaBaseOper extends AbstractOperator { messageAH = new AttributeHelper("message"); protected List topics = new ArrayList(); protected KafkaClient client = null; + protected SimpleConsumerClient simpleClient = null; private final Logger trace = Logger.getLogger(KafkaBaseOper.class .getCanonicalName()); @@ -70,8 +71,15 @@ public void initSchema(StreamSchema ss) throws Exception { supportedTypes.remove(MetaType.BLOB); topicAH.initialize(ss, false, supportedTypes); - trace.log(TraceLevel.INFO, "Creating client"); - client = new KafkaClient(topicAH, keyAH, messageAH, finalProperties); + + + OperatorContext operContext = getOperatorContext(); + if (operContext.getParameterNames().contains("partition") == false){ + //we don't need to create this client if using simpleConsumerClient + trace.log(TraceLevel.INFO, "Creating client"); + client = new KafkaClient(topicAH, keyAH, messageAH, finalProperties); + } + } @Parameter(cardinality = -1, optional = true, description = "Specify a Kafka property \\\"key=value\\\" form. " @@ -94,8 +102,9 @@ public void setPropertiesFile(String value) { } public String getPropertiesFile() { - - File file = new File(propertiesFile); + trace.log(TraceLevel.TRACE, "Properties file: " + propertiesFile); + if (propertiesFile == null) return null; + File file = new File(propertiesFile); // if the properties file is relative, the path is relative to the application directory if (!file.isAbsolute()) diff --git a/com.ibm.streamsx.messaging/impl/java/src/com/ibm/streamsx/messaging/kafka/KafkaSource.java b/com.ibm.streamsx.messaging/impl/java/src/com/ibm/streamsx/messaging/kafka/KafkaSource.java index abd83a4..1ca715e 100644 --- a/com.ibm.streamsx.messaging/impl/java/src/com/ibm/streamsx/messaging/kafka/KafkaSource.java +++ b/com.ibm.streamsx.messaging/impl/java/src/com/ibm/streamsx/messaging/kafka/KafkaSource.java @@ -6,6 +6,7 @@ package com.ibm.streamsx.messaging.kafka; +import java.io.IOException; import java.util.List; import java.util.logging.Logger; @@ -18,17 +19,20 @@ import com.ibm.streams.operator.model.OutputPorts; import com.ibm.streams.operator.model.Parameter; import com.ibm.streams.operator.model.PrimitiveOperator; +import com.ibm.streams.operator.state.Checkpoint; import com.ibm.streams.operator.state.ConsistentRegionContext; +import com.ibm.streams.operator.state.StateHandler; @OutputPorts(@OutputPortSet(cardinality=1, optional=false, description="Messages received from Kafka are sent on this output port.")) @PrimitiveOperator(name=KafkaSource.OPER_NAME, description=KafkaSource.DESC) @Icons(location16="icons/KafkaConsumer_16.gif", location32="icons/KafkaConsumer_32.gif") -public class KafkaSource extends KafkaBaseOper { +public class KafkaSource extends KafkaBaseOper implements StateHandler{ static final String OPER_NAME = "KafkaConsumer"; private int threadsPerTopic = 1; - + private int a_partition = -1; + private int triggerCount = -1; private static Logger trace = Logger.getLogger(KafkaSource.class.getName()); //consistent region checks @@ -36,28 +40,79 @@ public class KafkaSource extends KafkaBaseOper { public static void checkInConsistentRegion(OperatorContextChecker checker) { ConsistentRegionContext consistentRegionContext = checker.getOperatorContext().getOptionalContext(ConsistentRegionContext.class); - + OperatorContext operContext = checker.getOperatorContext(); + if(consistentRegionContext != null ) { - checker.setInvalidContext( OPER_NAME + " operator cannot be used inside a consistent region.", - new String[] {}); + //checker.setInvalidContext( OPER_NAME + " operator cannot be used inside a consistent region.", + // new String[] {}); + if (!operContext.getParameterNames().contains("partition")){ + checker.setInvalidContext("The partition parameter must be specified in consistent regions.", new String[] {}); + } } } - + + //simple consumer client checks + @ContextCheck(runtime = false, compile = true) + public static void checkCompileCompatability(OperatorContextChecker checker) { + OperatorContext operContext = checker.getOperatorContext(); + + if (!operContext.getParameterNames().contains("propertiesFile") + && !operContext.getParameterNames().contains("kafkaProperty")){ + checker.setInvalidContext("Missing properties: Neither propertiesFile nor kafkaProperty parameters are set. At least one must be set.",new String[] {}); + + } + + } + + //simple consumer client checks + @ContextCheck(runtime = true, compile = false) + public static void checkRuntimeCompatability(OperatorContextChecker checker) { + OperatorContext operContext = checker.getOperatorContext(); + + if (operContext.getParameterNames().contains("partition")){ + + if (operContext.getParameterValues("topic").size() > 1){ + checker.setInvalidContext("Invalid topic parameter: Only one topic can be specified when the partition parameter is set.", new String[] {}); + throw new IllegalArgumentException("Invalid topic parameter: Only one topic can be specified when the partition parameter is set."); + } + + } + + } + @Override public void initialize(OperatorContext context) throws Exception { super.initialize(context); super.initSchema(getOutput(0).getStreamSchema()); - + + + getOutput(0); if(threadsPerTopic < 1) - throw new IllegalArgumentException("Number of threads per topic cannot be less than one: " + threadsPerTopic); + throw new IllegalArgumentException("Number of threads per topic cannot be less than one: " + threadsPerTopic); + ConsistentRegionContext crContext = getOperatorContext().getOptionalContext(ConsistentRegionContext.class); + if( crContext != null){ + + if (a_partition == -1){ + throw new IllegalArgumentException("Partition parameter must be specified when using consistent region"); + } + + } } @Override public void allPortsReady() throws Exception { //initialize the client - trace.log(TraceLevel.INFO, "Initializing client"); - client.initConsumer(getOutput(0), getOperatorContext().getThreadFactory(), topics, threadsPerTopic); + trace.log(TraceLevel.INFO, "Initializing client"); + if(a_partition >= 0){ + trace.log(TraceLevel.INFO, "Using simple consumer client."); + simpleClient = new SimpleConsumerClient(topics.get(0), a_partition, topicAH, keyAH, messageAH, finalProperties, triggerCount); + simpleClient.initialize(getOperatorContext()); + simpleClient.allPortsReady(); + } else { + trace.log(TraceLevel.INFO, "Using high level consumer client."); + client.initConsumer(getOutput(0), getOperatorContext().getThreadFactory(), topics, threadsPerTopic); + } } @Parameter(name="threadsPerTopic", optional=true, @@ -70,7 +125,19 @@ public void setThreadsPerTopic(int value) { public void setTopic(List values) { if(values!=null) topics.addAll(values); - } + } + + @Parameter(name="partition", optional=true, + description="Partition to subscribe to.") + public void setPartition(int value) { + this.a_partition = value; + } + + @Parameter(name="triggerCount", optional=true, + description="Number of messages between checkpointing for consistent region. This is only relevant to operator driven checkpointing.") + public void setTriggerCount(int value) { + this.triggerCount = value; + } public static final String DESC = "This operator acts as a Kafka consumer receiving messages for one or more topics. " + @@ -78,7 +145,39 @@ public void setTopic(List values) { "Ordering of messages is not guaranteed." + "\\n\\n**Behavior in a Consistent Region**" + "\\nThis operator cannot be used inside a consistent region." - ; + ; + + @Override + public void close() throws IOException { + // TODO Auto-generated method stub + + } + + @Override + public void checkpoint(Checkpoint checkpoint) throws Exception { + simpleClient.checkpoint(checkpoint); + } + + @Override + public void drain() throws Exception { + simpleClient.drain(); + + } + + @Override + public void reset(Checkpoint checkpoint) throws Exception { + simpleClient.reset(checkpoint); + } + + @Override + public void resetToInitialState() throws Exception { + simpleClient.resetToInitialState(); + } + + @Override + public void retireCheckpoint(long id) throws Exception { + simpleClient.retireCheckpoint(id); + } } diff --git a/com.ibm.streamsx.messaging/impl/java/src/com/ibm/streamsx/messaging/kafka/KafkaSource2.java b/com.ibm.streamsx.messaging/impl/java/src/com/ibm/streamsx/messaging/kafka/KafkaSource2.java deleted file mode 100644 index 5b179ed..0000000 --- a/com.ibm.streamsx.messaging/impl/java/src/com/ibm/streamsx/messaging/kafka/KafkaSource2.java +++ /dev/null @@ -1,533 +0,0 @@ -/* Generated by Streams Studio: May 26, 2015 1:20:31 PM EDT */ -package com.ibm.streamsx.messaging.kafka; - - -import kafka.api.FetchRequest; -import kafka.api.FetchRequestBuilder; -import kafka.api.PartitionOffsetRequestInfo; -import kafka.common.ErrorMapping; -import kafka.common.TopicAndPartition; -import kafka.javaapi.*; -import kafka.javaapi.consumer.SimpleConsumer; -import kafka.message.MessageAndOffset; - -import java.io.File; -import java.io.FileReader; -import java.io.IOException; -import java.net.URI; -import java.nio.ByteBuffer; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Properties; - -import org.apache.log4j.Logger; - -import scala.actors.threadpool.Arrays; - -import com.ibm.streams.operator.AbstractOperator; -import com.ibm.streams.operator.OperatorContext; -import com.ibm.streams.operator.OutputTuple; -import com.ibm.streams.operator.StreamingOutput; -import com.ibm.streams.operator.log4j.TraceLevel; -import com.ibm.streams.operator.model.OutputPortSet; -import com.ibm.streams.operator.model.OutputPortSet.WindowPunctuationOutputMode; -import com.ibm.streams.operator.model.Libraries; -import com.ibm.streams.operator.model.OutputPorts; -import com.ibm.streams.operator.model.Parameter; -import com.ibm.streams.operator.model.PrimitiveOperator; -import com.ibm.streams.operator.state.Checkpoint; -import com.ibm.streams.operator.state.ConsistentRegionContext; -import com.ibm.streams.operator.state.StateHandler; - -@Libraries({ "opt/downloaded/*" }) -@PrimitiveOperator(name="KafkaSource2", namespace="com.ibm.streamsx.messaging.kafka", -description="Java Operator KafkaSource2") -@OutputPorts({@OutputPortSet(description="Port that produces tuples", cardinality=1, optional=false, windowPunctuationOutputMode=WindowPunctuationOutputMode.Generating), @OutputPortSet(description="Optional output ports", optional=true, windowPunctuationOutputMode=WindowPunctuationOutputMode.Generating)}) -public class KafkaSource2 extends AbstractOperator implements StateHandler { - - private static Logger TRACE = Logger.getLogger(KafkaSource2.class.getName()); - private Thread processThread; - static final String OPER_NAME = "KafkaConsumer"; - private ConsistentRegionContext crContext; - private boolean shutdown = false; - private long triggerCount; - private long triggerIteration = 0; - //consumer variables - public SimpleConsumer myConsumer = null; - String a_topic; - String leadBroker; - int a_port; - long readOffset; - long resetReadOffset = -1; - int a_partition; - int so_timeout=10000; - String clientName; - protected String propertiesFile = null; - List m_replicaBrokers = new ArrayList(); - protected Properties properties = new Properties(), - finalProperties = new Properties(); - - - @Override - public synchronized void initialize(OperatorContext context) - throws Exception { - - super.initialize(context); - Logger.getLogger(this.getClass()).trace("Operator " + context.getName() + " initializing in PE: " + context.getPE().getPEId() + " in Job: " + context.getPE().getJobId() ); - crContext = context.getOptionalContext(ConsistentRegionContext.class); - TRACE.log(TraceLevel.TRACE, "Beginning consumer initialization"); - String propFile = getPropertiesFile(); - if (propFile != null) { - finalProperties.load(new FileReader(propFile)); - } - finalProperties.putAll(properties); - if (finalProperties == null || finalProperties.isEmpty()) - throw new Exception( - "Kafka connection properties must be specified."); - TRACE.log(TraceLevel.TRACE, "Final properties: " + finalProperties.toString()); - - //name client - clientName = a_topic + "_partition_" + Integer.toString(a_partition) + "_pe_" + context.getPE().getPEId(); - TRACE.log(TraceLevel.TRACE, "Initializing consumer with clientName: " + clientName); - - String brokerString = finalProperties.getProperty("metadata.broker.list"); - List hostAndPortStrings = Arrays.asList(brokerString.split(",")); - - PartitionMetadata metadata = findLeader(hostAndPortStrings, a_topic, a_partition); - if (metadata == null) { - TRACE.log(TraceLevel.ERROR, "Can't find metadata for Topic and Partition. Exiting."); - shutdown(); - return; - } - if (metadata.leader() == null) { - TRACE.log(TraceLevel.ERROR, "Can't find Leader for Topic and Partition. Exiting."); - shutdown(); - return; - } - - TRACE.log(TraceLevel.INFO, "BrokerString: " + brokerString + " Broker List: " + hostAndPortStrings.toString() + " Leader: " + metadata.leader()); - - leadBroker = metadata.leader().host(); - a_port = metadata.leader().port(); - - myConsumer = new SimpleConsumer(leadBroker, a_port, so_timeout, 64 * 1024, clientName); - TRACE.log(TraceLevel.TRACE, "Consumer initialization complete."); - - readOffset = getLastOffset(myConsumer, a_topic, a_partition, - kafka.api.OffsetRequest.LatestTime(), clientName); - - - /* - * Create the thread for producing tuples. - * The thread is created at initialize time but started. - * The thread will be started by allPortsReady(). - */ - processThread = getOperatorContext().getThreadFactory().newThread( - new Runnable() { - - @Override - public void run() { - try { - produceTuples(); - } catch (Exception e) { - Logger.getLogger(this.getClass()).error("Operator error", e); //$NON-NLS-1$ - } - } - - }); - - /* - * Set the thread not to be a daemon to ensure that the SPL runtime - * will wait for the thread to complete before determining the - * operator is complete. - */ - processThread.setDaemon(false); - } - - /** - * Notification that initialization is complete and all input and output ports - * are connected and ready to receive and submit tuples. - * @throws Exception Operator failure, will cause the enclosing PE to terminate. - */ - @Override - public synchronized void allPortsReady() throws Exception { - OperatorContext context = getOperatorContext(); - Logger.getLogger(this.getClass()).trace("Operator " + context.getName() + " all ports are ready in PE: " + context.getPE().getPEId() + " in Job: " + context.getPE().getJobId() ); - // Start a thread for producing tuples because operator - // implementations must not block and must return control to the caller. - processThread.start(); - } - - - /** - * Submit new tuples to the output stream - * @throws Exception if an error occurs while submitting a tuple - */ - private void produceTuples() throws Exception { - final StreamingOutput out = getOutput(0); - - - OutputTuple tuple = out.newTuple(); - long numRead; - long currentOffset; - FetchResponse fetchResponse; - - - while(!shutdown){ - //TRACE.log(TraceLevel.TRACE, "Iteration through the while loop. ReadOffset: " + Long.toString(readOffset)); - - if (myConsumer == null){ - myConsumer = new SimpleConsumer(leadBroker, a_port, so_timeout, 64*1024, clientName); - } - - FetchRequest req = new FetchRequestBuilder() - .clientId(clientName) - .addFetch(a_topic, a_partition, - readOffset, 100000).build(); -// System.out.println("Fetched a request! ReadOffset: " + readOffset); - //TRACE.log(TraceLevel.TRACE, "FetchRequest:" + req.toString()); - - try{ - fetchResponse = myConsumer.fetch(req); - //TRACE.log(TraceLevel.TRACE, "Fetched new response:" + fetchResponse.toString()); - } catch(Exception e) { - //System.out.println("Catching exception " + e); - TRACE.log(TraceLevel.ERROR, "Fetch error. Lead server cannot be contacted. Exception: " + e.getStackTrace()); - myConsumer.close(); - myConsumer = null; - fetchResponse = null; - leadBroker = findNewLeader(leadBroker, a_topic, a_partition); - } - - - - if (fetchResponse != null) { - - if (fetchResponse.hasError()){ - // Something went wrong! - short code = fetchResponse.errorCode(a_topic, a_partition); - TRACE.log(TraceLevel.ERROR, "Error fetching data from the Broker:" + leadBroker + " Reason: " + code); - - if (code == ErrorMapping.OffsetOutOfRangeCode()) { - // We asked for an invalid offset. This should never happen. - TRACE.log(TraceLevel.ERROR, "Tried to request an invalid offset. Exiting."); - } - myConsumer.close(); - myConsumer = null; - leadBroker = findNewLeader(leadBroker, a_topic, a_partition); - } else { - numRead = 0; - for (MessageAndOffset messageAndOffset : fetchResponse.messageSet( - a_topic, a_partition)) { - // TRACE.log(TraceLevel.INFO, - // "Looping through messageAndOffset."); - - currentOffset = messageAndOffset.offset(); - tuple = out.newTuple(); - if (currentOffset < readOffset) { - TRACE.log(TraceLevel.ERROR, "Found an old offset: " - + currentOffset + " Expecting: " + readOffset); - } - - ByteBuffer messagePayload = messageAndOffset.message() - .payload(); - byte[] messageBytes = new byte[messagePayload.limit()]; - messagePayload.get(messageBytes); - String message = new String(messageBytes, "UTF-8"); - ByteBuffer keyPayload = messageAndOffset.message().key(); - byte[] keyBytes = new byte[keyPayload.limit()]; - keyPayload.get(keyBytes); - String key = new String(keyBytes, "UTF-8"); - - // Set attributes in tuple: - tuple.setString("message", message); - tuple.setString("key", key); - - try { - if (crContext != null) { - crContext.acquirePermit(); - } - - //if there has been a reset, we NEED to get out of this loop and do a new fetch request - if (resetReadOffset != -1){ - readOffset = resetReadOffset; - resetReadOffset = -1; - break; - } - -// System.out.println( -// "Submitting tuple from offset: " + readOffset + " : " + -// message + " key: " + key); - numRead++; - // Submit tuple to output stream - out.submit(tuple); - readOffset = messageAndOffset.nextOffset(); - - if (crContext != null && crContext.isTriggerOperator()) { - triggerIteration++; - if (triggerIteration >= triggerCount) { - // TRACE.log(TraceLevel.INFO, - // "Trigger fired. Making consistent."); - crContext.makeConsistent(); - triggerIteration = 0; - } - } - } catch (Exception e) { - TRACE.log(TraceLevel.ERROR, - "Unexpected exception: " + e.toString()); - } finally { - // release permit when done submitting - if (crContext != null) { - // TRACE.log(TraceLevel.INFO, "Releasing permit."); - crContext.releasePermit(); - } - } - } - if (numRead == 0) { - try { - Thread.sleep(50); - } catch (InterruptedException ie) { - TRACE.log(TraceLevel.ERROR, "Exception while sleeping: " + ie); - } - } - - - } - - - } - } - if (myConsumer != null) myConsumer.close(); - } - - /** - * Shutdown this operator, which will interrupt the thread - * executing the produceTuples() method. - * @throws Exception Operator failure, will cause the enclosing PE to terminate. - */ - public synchronized void shutdown() throws Exception { - shutdown = true; - if (processThread != null) { - processThread.interrupt(); - processThread = null; - } - OperatorContext context = getOperatorContext(); - Logger.getLogger(this.getClass()).trace("Operator " + context.getName() + " shutting down in PE: " + context.getPE().getPEId() + " in Job: " + context.getPE().getJobId() ); - - // Must call super.shutdown() - super.shutdown(); - } - - @Override - public void close() throws IOException { - TRACE.log(TraceLevel.INFO, "StateHandler close"); - } - - @Override - public void checkpoint(Checkpoint checkpoint) throws Exception { -// System.out.println("Checkpoint readOffset " + readOffset); - TRACE.log(TraceLevel.INFO, "Checkpoint " + checkpoint.getSequenceId()); - checkpoint.getOutputStream().writeLong(readOffset); - } - - @Override - public void drain() throws Exception { -// System.out.println("Drain..."); - TRACE.log(TraceLevel.INFO, "Drain..."); - } - @Override - public void reset(Checkpoint checkpoint) throws Exception { -// System.out.println("Reset"); - TRACE.log(TraceLevel.INFO, "Reset to checkpoint " + checkpoint.getSequenceId()); - resetReadOffset = checkpoint.getInputStream().readLong(); - -// System.out.println("Set readOffset to : " + readOffset); - } - @Override - public void resetToInitialState() throws Exception { -// System.out.println("ResetToInitial"); - TRACE.log(TraceLevel.INFO, "Reset to initial state"); - } - - @Override - public void retireCheckpoint(long id) throws Exception { -// System.out.println("Retire checkpoint"); - TRACE.log(TraceLevel.INFO, "Retire checkpoint"); - } - - /** - * - * @param hostAndPortStrings - * @param topic - * @param partition - * @return - */ - private PartitionMetadata findLeader(List hostAndPortStrings, - String topic, int partition) { - PartitionMetadata returnMetaData = null; - loop: for (String seed : hostAndPortStrings) { - SimpleConsumer consumer = null; - try { - URI uri = new URI("my://" + seed); // may throw URISyntaxException - String host = uri.getHost(); - int port = uri.getPort(); - consumer = new SimpleConsumer(host, port, so_timeout, 64 * 1024, - "leaderLookup"); - List topics = Collections.singletonList(topic); - TopicMetadataRequest req = new TopicMetadataRequest(topics); - kafka.javaapi.TopicMetadataResponse resp = consumer.send(req); - - List metaData = resp.topicsMetadata(); - for (TopicMetadata item : metaData) { - for (PartitionMetadata part : item.partitionsMetadata()) { - if (part.partitionId() == partition) { - returnMetaData = part; - break loop; - } - } - } - } catch (Exception e) { - System.out.println("Error communicating with Broker [" + seed - + "] to find Leader for [" + topic + ", " - + partition + "] Reason: " + e); - TRACE.log(TraceLevel.ERROR, "Error communicating with Broker [" + seed - + "] to find Leader for [" + topic + ", " - + partition + "] Reason: " + e); - } finally { - if (consumer != null) - consumer.close(); - } - } - if (returnMetaData != null) { - m_replicaBrokers.clear(); - for (kafka.cluster.Broker replica : returnMetaData.replicas()) { - String rep = replica.host() + ":" + replica.port(); - TRACE.log(TraceLevel.TRACE, "Adding replica: " + rep); - m_replicaBrokers.add(rep); - } - } - return returnMetaData; - } - - /** - * - * @param a_oldLeader - * @param topic - * @param partition - * @return - */ - private String findNewLeader(String a_oldLeader, String topic, - int partition) throws Exception { - for (int i = 0; i < 3; i++) { - boolean goToSleep = false; - PartitionMetadata metadata = findLeader(m_replicaBrokers, - topic, partition); - if (metadata == null) { - goToSleep = true; - } else if (metadata.leader() == null) { - goToSleep = true; - } else if (a_oldLeader.equalsIgnoreCase(metadata.leader().host()) - && i == 0) { - // first time through if the leader hasn't changed give - // ZooKeeper a second to recover - // second time, assume the broker did recover before failover, - // or it was a non-Broker issue - // - goToSleep = true; - } else { - return metadata.leader().host(); - } - if (goToSleep) { - try { - Thread.sleep(1000); - } catch (InterruptedException ie) { - } - } - } - System.out - .println("Unable to find new leader after Broker failure. Exiting"); - TRACE.log(TraceLevel.ERROR, "Can't find Leader for Topic and Partition. Exiting."); - throw new Exception( - "Unable to find new leader after Broker failure. Exiting"); - } - - public static long getLastOffset(SimpleConsumer consumer, String topic, - int partition, long whichTime, String clientName) { - TopicAndPartition topicAndPartition = new TopicAndPartition(topic, - partition); - Map requestInfo = new HashMap(); - requestInfo.put(topicAndPartition, new PartitionOffsetRequestInfo( - whichTime, 1)); - kafka.javaapi.OffsetRequest request = new kafka.javaapi.OffsetRequest( - requestInfo, kafka.api.OffsetRequest.CurrentVersion(), - clientName); - OffsetResponse response = consumer.getOffsetsBefore(request); - - if (response.hasError()) { - TRACE.log(TraceLevel.ERROR, "Error fetching data Offset Data the Broker. Reason: " - + response.errorCode(topic, partition)); - return 0; - } - long[] offsets = response.offsets(topic, partition); - TRACE.log(TraceLevel.TRACE, "Retrieving offsets: " + Arrays.toString(offsets)); - return offsets[0]; - } - - public String getPropertiesFile() { - TRACE.log(TraceLevel.TRACE, "Properties file: " + propertiesFile); - if (propertiesFile == null) return null; - File file = new File(propertiesFile); - - // if the properties file is relative, the path is relative to the application directory - if (!file.isAbsolute()) - { - propertiesFile = getOperatorContext().getPE().getApplicationDirectory().getAbsolutePath() + "/" + propertiesFile; - } - return propertiesFile; - } - - @Parameter(optional = true, description = "Properties file containing kafka properties. Properties file is recommended to be stored in the etc directory. If a relative path is specified, the path is relative to the application directory.") - public void setPropertiesFile(String value) { - this.propertiesFile = value; - } - - @Parameter(name="topic", optional=false, - description="Topic to be subscribed to.") - public void setTopic(String value) { - if(value!=null) - this.a_topic = value; - } - - @Parameter(name="partition", optional=false, - description="Partition to subscribe to.") - public void setPartition(int value) { - this.a_partition = value; - } - - @Parameter(name="triggerCount", optional=true, - description="Number of messages between checkpointing for consistent region. This is only relevant to operator driven checkpointing.") - public void setTriggerCount(int value) { - this.triggerCount = value; - } - - @Parameter(cardinality = -1, optional = true, description = "Specify a Kafka property \\\"key=value\\\" form. " - + "This will override any property specified in the properties file.") - public void setKafkaProperty(List values) { - for (String value : values) { - int idx = value.indexOf("="); - if (idx == -1) - throw new IllegalArgumentException("Invalid property: " + value - + ", not in the key=value format"); - String name = value.substring(0, idx); - String v = value.substring(idx + 1, value.length()); - properties.setProperty(name, v); - } - } - - -} - - diff --git a/com.ibm.streamsx.messaging/impl/java/src/com/ibm/streamsx/messaging/kafka/SimpleConsumerClient.java b/com.ibm.streamsx.messaging/impl/java/src/com/ibm/streamsx/messaging/kafka/SimpleConsumerClient.java new file mode 100644 index 0000000..f12077a --- /dev/null +++ b/com.ibm.streamsx.messaging/impl/java/src/com/ibm/streamsx/messaging/kafka/SimpleConsumerClient.java @@ -0,0 +1,492 @@ +/* Generated by Streams Studio: May 26, 2015 1:20:31 PM EDT */ +package com.ibm.streamsx.messaging.kafka; + +import kafka.api.FetchRequest; +import kafka.api.FetchRequestBuilder; +import kafka.api.PartitionOffsetRequestInfo; +import kafka.cluster.Broker; +import kafka.common.ErrorMapping; +import kafka.common.TopicAndPartition; +import kafka.javaapi.*; +import kafka.javaapi.consumer.SimpleConsumer; +import kafka.message.MessageAndOffset; +import kafka.utils.ZkUtils; +import kafka.utils.ZKStringSerializer$; +import org.I0Itec.zkclient.ZkClient; + + +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Properties; + +import org.apache.log4j.Logger; + +import scala.actors.threadpool.Arrays; + + +import com.ibm.streams.operator.OperatorContext; +import com.ibm.streams.operator.OutputTuple; +import com.ibm.streams.operator.StreamingOutput; +import com.ibm.streams.operator.log4j.TraceLevel; +import com.ibm.streams.operator.state.Checkpoint; +import com.ibm.streams.operator.state.ConsistentRegionContext; +import com.ibm.streams.operator.state.StateHandler; + +public class SimpleConsumerClient implements StateHandler { + + private static Logger TRACE = Logger.getLogger(SimpleConsumerClient.class + .getCanonicalName()); + private Thread processThread; + static final String OPER_NAME = "KafkaConsumer"; + private ConsistentRegionContext crContext; + private boolean shutdown = false; + private long triggerCount; + private long triggerIteration = 0; + private OperatorContext operContext; + // consumer variables + public SimpleConsumer myConsumer = null; + ZkClient zkClient; + String topic; + Broker leadBroker; + long readOffset; + long resetReadOffset = -1; + int partition; + int so_timeout; + String clientName; + protected Properties finalProperties = new Properties(); + + public SimpleConsumerClient(String a_topic, int a_partition, + AttributeHelper topicAH, AttributeHelper keyAH, + AttributeHelper messageAH, Properties props, long trigCnt) { + this.topic = a_topic; + // this.keyAH = keyAH; + // this.messageAH = messageAH; + this.partition = a_partition; + this.finalProperties = props; + this.triggerCount = trigCnt; + } + + public void initialize(OperatorContext context) throws Exception { + + // super.initialize(context); + Logger.getLogger(this.getClass()).trace( + "Operator " + context.getName() + " initializing in PE: " + + context.getPE().getPEId() + " in Job: " + + context.getPE().getJobId()); + operContext = context; + crContext = context.getOptionalContext(ConsistentRegionContext.class); + TRACE.log(TraceLevel.TRACE, "Beginning consumer initialization"); + + // name client + clientName = topic + "_partition_" + Integer.toString(partition) + + "_pe_" + context.getPE().getPEId(); + TRACE.log(TraceLevel.TRACE, "Initializing consumer with clientName: " + + clientName); + + //default 30 seconds + so_timeout = Integer.parseInt(finalProperties.getProperty("socket.timeout.ms", "30000")); + + zkClient = getInitializedZkClient(); + leadBroker = findLeader(zkClient, topic, partition); + + if (leadBroker == null) { + TRACE.log(TraceLevel.ERROR, + "Can't find Leader for Topic and Partition. Exiting."); + shutdown(); + return; + } + + myConsumer = new SimpleConsumer(leadBroker.host(), leadBroker.port(), + so_timeout, 64 * 1024, clientName); + TRACE.log(TraceLevel.TRACE, "Consumer initialization complete."); + + readOffset = getLastOffset(myConsumer, topic, partition, + kafka.api.OffsetRequest.LatestTime(), clientName); + + /* + * Create the thread for producing tuples. The thread is created at + * initialize time but started. The thread will be started by + * allPortsReady(). + */ + processThread = context.getThreadFactory().newThread(new Runnable() { + + @Override + public void run() { + try { + produceTuples(); + } catch (Exception e) { + Logger.getLogger(this.getClass()) + .error("Operator error", e); //$NON-NLS-1$ + } + } + + }); + + /* + * Set the thread not to be a daemon to ensure that the SPL runtime will + * wait for the thread to complete before determining the operator is + * complete. + */ + processThread.setDaemon(false); + } + + /** + * Notification that initialization is complete and all input and output + * ports are connected and ready to receive and submit tuples. + * + * @throws Exception + * Operator failure, will cause the enclosing PE to terminate. + */ + public synchronized void allPortsReady() throws Exception { + Logger.getLogger(this.getClass()).trace( + "Operator " + operContext.getName() + + " all ports are ready in PE: " + + operContext.getPE().getPEId() + " in Job: " + + operContext.getPE().getJobId()); + // Start a thread for producing tuples because operator + // implementations must not block and must return control to the caller. + processThread.start(); + } + + /** + * Submit new tuples to the output stream + * + * @throws Exception + * if an error occurs while submitting a tuple + */ + private void produceTuples() throws Exception { + final StreamingOutput out = operContext + .getStreamingOutputs().get(0); + OutputTuple tuple = out.newTuple(); + long numRead; + long currentOffset; + FetchResponse fetchResponse; + + while (!shutdown) { + TRACE.log( + TraceLevel.TRACE, + "Iteration through the while loop. ReadOffset: " + + Long.toString(readOffset)); + + if (myConsumer == null) { + myConsumer = new SimpleConsumer(leadBroker.host(), + leadBroker.port(), so_timeout, 64 * 1024, clientName); + } + + FetchRequest req = new FetchRequestBuilder().clientId(clientName) + .addFetch(topic, partition, readOffset, 100000).build(); + + try { + fetchResponse = myConsumer.fetch(req); + } catch (Exception e) { + // System.out.println("Catching exception " + e); + TRACE.log(TraceLevel.ERROR, + "Fetch error. Lead server cannot be contacted. Exception: " + + e.getStackTrace()); + myConsumer.close(); + myConsumer = null; + fetchResponse = null; + leadBroker = findNewLeader(leadBroker, zkClient, topic, + partition); + } + + if (fetchResponse != null) { + + if (fetchResponse.hasError()) { + // Something went wrong! + handleFetchError(fetchResponse); + } else { + numRead = 0; + for (MessageAndOffset messageAndOffset : fetchResponse + .messageSet(topic, partition)) { + currentOffset = messageAndOffset.offset(); + tuple = out.newTuple(); + + if (currentOffset < readOffset) { + TRACE.log(TraceLevel.ERROR, "Found an old offset: " + + currentOffset + " Expecting: " + + readOffset); + } + + String message = getMessageFromMessageAndOffset(messageAndOffset); + String key = getKeyFromMessageAndOffset(messageAndOffset); + + // Set attributes in tuple: + tuple.setString("message", message); + tuple.setString("key", key); + + try { + if (crContext != null) { + crContext.acquirePermit(); + } + + // if there has been a reset, we NEED to get out of + // this loop and do a new fetch request + if (resetReadOffset != -1) { + readOffset = resetReadOffset; + resetReadOffset = -1; + break; + } + + numRead++; + // Submit tuple to output stream + out.submit(tuple); + readOffset = messageAndOffset.nextOffset(); + + if (crContext != null + && crContext.isTriggerOperator()) { + triggerIteration++; + if (triggerIteration >= triggerCount) { + + crContext.makeConsistent(); + triggerIteration = 0; + } + } + } catch (Exception e) { + TRACE.log(TraceLevel.ERROR, + "Unexpected exception: " + e.toString()); + } finally { + // release permit when done submitting + if (crContext != null) { + crContext.releasePermit(); + } + } + } + if (numRead == 0) { + try { + Thread.sleep(50); + } catch (InterruptedException ie) { + TRACE.log(TraceLevel.ERROR, + "Exception while sleeping: " + ie); + } + } + + } + + } + } + if (myConsumer != null) + myConsumer.close(); + } + + private String getKeyFromMessageAndOffset(MessageAndOffset messageAndOffset) throws UnsupportedEncodingException { + ByteBuffer keyPayload = messageAndOffset.message() + .key(); + byte[] keyBytes = new byte[keyPayload.limit()]; + keyPayload.get(keyBytes); + String key = new String(keyBytes, "UTF-8"); + return key; + } + + private String getMessageFromMessageAndOffset( + MessageAndOffset messageAndOffset) throws UnsupportedEncodingException { + + ByteBuffer messagePayload = messageAndOffset.message() + .payload(); + + byte[] messageBytes = new byte[messagePayload.limit()]; + messagePayload.get(messageBytes); + String message = new String(messageBytes, "UTF-8"); + return message; + } + + private void handleFetchError(FetchResponse fetchResponse) { + short code = fetchResponse.errorCode(topic, partition); + TRACE.log( + TraceLevel.ERROR, + "Error fetching data from the Broker:" + + leadBroker.host() + " Reason: " + code); + + if (code == ErrorMapping.OffsetOutOfRangeCode()) { + // We asked for an invalid offset. This should never + // happen. + TRACE.log(TraceLevel.ERROR, + "Tried to request an invalid offset. Exiting."); + } + myConsumer.close(); + myConsumer = null; + leadBroker = findLeader(zkClient, topic, partition); + } + + /** + * Shutdown this operator, which will interrupt the thread executing the + * produceTuples() method. + * + * @throws Exception + * Operator failure, will cause the enclosing PE to terminate. + */ + public synchronized void shutdown() throws Exception { + shutdown = true; + if (processThread != null) { + processThread.interrupt(); + processThread = null; + } + // OperatorContext context = getOperatorContext(); + Logger.getLogger(this.getClass()).trace( + "Operator " + operContext.getName() + " shutting down in PE: " + + operContext.getPE().getPEId() + " in Job: " + + operContext.getPE().getJobId()); + + // Must call super.shutdown() + // super.shutdown(); + } + + @Override + public void close() throws IOException { + TRACE.log(TraceLevel.INFO, "StateHandler close"); + } + + @Override + public void checkpoint(Checkpoint checkpoint) throws Exception { + // System.out.println("Checkpoint readOffset " + readOffset); + TRACE.log(TraceLevel.INFO, "Checkpoint " + checkpoint.getSequenceId()); + checkpoint.getOutputStream().writeLong(readOffset); + } + + @Override + public void drain() throws Exception { + // System.out.println("Drain..."); + TRACE.log(TraceLevel.INFO, "Drain..."); + } + + @Override + public void reset(Checkpoint checkpoint) throws Exception { + // System.out.println("Reset"); + TRACE.log(TraceLevel.INFO, + "Reset to checkpoint " + checkpoint.getSequenceId()); + resetReadOffset = checkpoint.getInputStream().readLong(); + + // System.out.println("Set readOffset to : " + readOffset); + } + + @Override + public void resetToInitialState() throws Exception { + // System.out.println("ResetToInitial"); + TRACE.log(TraceLevel.INFO, "Reset to initial state"); + } + + @Override + public void retireCheckpoint(long id) throws Exception { + // System.out.println("Retire checkpoint"); + TRACE.log(TraceLevel.INFO, "Retire checkpoint"); + } + + private ZkClient getInitializedZkClient() { + ZkClient localZkClient; + String zkConnect = finalProperties.getProperty("zookeeper.connect"); + int zkSessionTimeout = getIntegerProperty( + "zookeeper.session.timeout.ms", 400); + int zkConnectionTimeout = getIntegerProperty("zookeeper.sync.time.ms", + 200); + + TRACE.log(TraceLevel.INFO, + "Initializing ZooKeeper with values: zkConnect(" + zkConnect + + ") zkSessionTimeout(" + zkSessionTimeout + + ") zkConnectionTimeout(" + zkConnectionTimeout + ")"); + + try { + localZkClient = new ZkClient(zkConnect, zkSessionTimeout, + zkConnectionTimeout, ZKStringSerializer$.MODULE$); + } catch (Exception e) { + TRACE.log(TraceLevel.ERROR, + "Zookeeper client did not initialize correctly with exception: " + + e); + throw e; + } + + return localZkClient; + } + + private int getIntegerProperty(String propName, int defaultVal) { + int integerProp = defaultVal; + try { + integerProp = Integer.parseInt(finalProperties + .getProperty(propName)); + } catch (Exception e){ + e.printStackTrace(); + TRACE.log(TraceLevel.ERROR, "Property " + propName + "was not input as type int. Exception: " + e); + } + + return integerProp; + } + + private Broker findLeader(ZkClient a_zkClient, String a_topic, + int a_partition) { + int leaderID; + Broker a_leadBroker = null; + + try { + + leaderID = ZkUtils.getLeaderForPartition(zkClient, a_topic, + a_partition).get(); + a_leadBroker = ZkUtils.getBrokerInfo(zkClient, leaderID).get(); + + } catch (Exception e) { + + e.printStackTrace(); + System.out.println("Exception from ZkClient!"); + + } + + return a_leadBroker; + } + + private Broker findNewLeader(Broker oldBroker, ZkClient a_zkClient, + String a_topic, int a_partition) { + + for (int i = 0; i < 3; i++) { + boolean goSleep = false; + Broker newLeadBroker = findLeader(a_zkClient, a_topic, a_partition); + + if (newLeadBroker == null) { + goSleep = true; + } else if (newLeadBroker.host().equalsIgnoreCase(oldBroker.host()) + && i == 0) { + // if it's the first time, let's give zookeeper a chance to + // recover. + goSleep = true; + } else { + return newLeadBroker; + } + + if (goSleep) { + try { + Thread.sleep(1000); + } catch (InterruptedException ie) { + } + } + } + + return null; + } + + public static long getLastOffset(SimpleConsumer consumer, String topic, + int partition, long whichTime, String clientName) { + TopicAndPartition topicAndPartition = new TopicAndPartition(topic, + partition); + Map requestInfo = new HashMap(); + requestInfo.put(topicAndPartition, new PartitionOffsetRequestInfo( + whichTime, 1)); + kafka.javaapi.OffsetRequest request = new kafka.javaapi.OffsetRequest( + requestInfo, kafka.api.OffsetRequest.CurrentVersion(), + clientName); + OffsetResponse response = consumer.getOffsetsBefore(request); + + if (response.hasError()) { + TRACE.log(TraceLevel.ERROR, + "Error fetching data Offset Data the Broker. Reason: " + + response.errorCode(topic, partition)); + return 0; + } + long[] offsets = response.offsets(topic, partition); + TRACE.log(TraceLevel.TRACE, + "Retrieving offsets: " + Arrays.toString(offsets)); + return offsets[0]; + } + +} From cb9132dd0b06fbaf28c8d254e1bf261d1fc613cd Mon Sep 17 00:00:00 2001 From: Alex-Cook4 Date: Fri, 10 Jul 2015 16:20:13 -0500 Subject: [PATCH 05/14] Style update --- .../messaging/kafka/SimpleConsumerClient.java | 426 +++++++++--------- 1 file changed, 213 insertions(+), 213 deletions(-) diff --git a/com.ibm.streamsx.messaging/impl/java/src/com/ibm/streamsx/messaging/kafka/SimpleConsumerClient.java b/com.ibm.streamsx.messaging/impl/java/src/com/ibm/streamsx/messaging/kafka/SimpleConsumerClient.java index f12077a..e42528a 100644 --- a/com.ibm.streamsx.messaging/impl/java/src/com/ibm/streamsx/messaging/kafka/SimpleConsumerClient.java +++ b/com.ibm.streamsx.messaging/impl/java/src/com/ibm/streamsx/messaging/kafka/SimpleConsumerClient.java @@ -41,6 +41,29 @@ public class SimpleConsumerClient implements StateHandler { private static Logger TRACE = Logger.getLogger(SimpleConsumerClient.class .getCanonicalName()); + public static long getLastOffset(SimpleConsumer consumer, String topic, + int partition, long whichTime, String clientName) { + TopicAndPartition topicAndPartition = new TopicAndPartition(topic, + partition); + Map requestInfo = new HashMap(); + requestInfo.put(topicAndPartition, new PartitionOffsetRequestInfo( + whichTime, 1)); + kafka.javaapi.OffsetRequest request = new kafka.javaapi.OffsetRequest( + requestInfo, kafka.api.OffsetRequest.CurrentVersion(), + clientName); + OffsetResponse response = consumer.getOffsetsBefore(request); + + if (response.hasError()) { + TRACE.log(TraceLevel.ERROR, + "Error fetching data Offset Data the Broker. Reason: " + + response.errorCode(topic, partition)); + return 0; + } + long[] offsets = response.offsets(topic, partition); + TRACE.log(TraceLevel.TRACE, + "Retrieving offsets: " + Arrays.toString(offsets)); + return offsets[0]; + } private Thread processThread; static final String OPER_NAME = "KafkaConsumer"; private ConsistentRegionContext crContext; @@ -50,14 +73,15 @@ public class SimpleConsumerClient implements StateHandler { private OperatorContext operContext; // consumer variables public SimpleConsumer myConsumer = null; - ZkClient zkClient; - String topic; - Broker leadBroker; - long readOffset; - long resetReadOffset = -1; - int partition; - int so_timeout; - String clientName; + private ZkClient zkClient; + private String topic; + private Broker leadBroker; + private long readOffset; + private long resetReadOffset = -1; + private int partition; + private int so_timeout; + private String clientName; + protected Properties finalProperties = new Properties(); public SimpleConsumerClient(String a_topic, int a_partition, @@ -71,6 +95,170 @@ public SimpleConsumerClient(String a_topic, int a_partition, this.triggerCount = trigCnt; } + /** + * Notification that initialization is complete and all input and output + * ports are connected and ready to receive and submit tuples. + * + * @throws Exception + * Operator failure, will cause the enclosing PE to terminate. + */ + public synchronized void allPortsReady() throws Exception { + Logger.getLogger(this.getClass()).trace( + "Operator " + operContext.getName() + + " all ports are ready in PE: " + + operContext.getPE().getPEId() + " in Job: " + + operContext.getPE().getJobId()); + // Start a thread for producing tuples because operator + // implementations must not block and must return control to the caller. + processThread.start(); + } + + @Override + public void checkpoint(Checkpoint checkpoint) throws Exception { + // System.out.println("Checkpoint readOffset " + readOffset); + TRACE.log(TraceLevel.INFO, "Checkpoint " + checkpoint.getSequenceId()); + checkpoint.getOutputStream().writeLong(readOffset); + } + + @Override + public void close() throws IOException { + TRACE.log(TraceLevel.INFO, "StateHandler close"); + } + + @Override + public void drain() throws Exception { + // System.out.println("Drain..."); + TRACE.log(TraceLevel.INFO, "Drain..."); + } + + private Broker findLeader(ZkClient a_zkClient, String a_topic, + int a_partition) { + int leaderID; + Broker a_leadBroker = null; + + try { + + leaderID = ZkUtils.getLeaderForPartition(zkClient, a_topic, + a_partition).get(); + a_leadBroker = ZkUtils.getBrokerInfo(zkClient, leaderID).get(); + + } catch (Exception e) { + + e.printStackTrace(); + System.out.println("Exception from ZkClient!"); + + } + + return a_leadBroker; + } + + private Broker findNewLeader(Broker oldBroker, ZkClient a_zkClient, + String a_topic, int a_partition) { + + for (int i = 0; i < 3; i++) { + boolean goSleep = false; + Broker newLeadBroker = findLeader(a_zkClient, a_topic, a_partition); + + if (newLeadBroker == null) { + goSleep = true; + } else if (newLeadBroker.host().equalsIgnoreCase(oldBroker.host()) + && i == 0) { + // if it's the first time, let's give zookeeper a chance to + // recover. + goSleep = true; + } else { + return newLeadBroker; + } + + if (goSleep) { + try { + Thread.sleep(1000); + } catch (InterruptedException ie) { + } + } + } + + return null; + } + + private ZkClient getInitializedZkClient() { + ZkClient localZkClient; + String zkConnect = finalProperties.getProperty("zookeeper.connect"); + int zkSessionTimeout = getIntegerProperty( + "zookeeper.session.timeout.ms", 400); + int zkConnectionTimeout = getIntegerProperty("zookeeper.sync.time.ms", + 200); + + TRACE.log(TraceLevel.INFO, + "Initializing ZooKeeper with values: zkConnect(" + zkConnect + + ") zkSessionTimeout(" + zkSessionTimeout + + ") zkConnectionTimeout(" + zkConnectionTimeout + ")"); + + try { + localZkClient = new ZkClient(zkConnect, zkSessionTimeout, + zkConnectionTimeout, ZKStringSerializer$.MODULE$); + } catch (Exception e) { + TRACE.log(TraceLevel.ERROR, + "Zookeeper client did not initialize correctly with exception: " + + e); + throw e; + } + + return localZkClient; + } + + private int getIntegerProperty(String propName, int defaultVal) { + int integerProp = defaultVal; + try { + integerProp = Integer.parseInt(finalProperties + .getProperty(propName)); + } catch (Exception e){ + e.printStackTrace(); + TRACE.log(TraceLevel.ERROR, "Property " + propName + " was not input as type int. Exception: " + e); + } + + return integerProp; + } + + private String getKeyFromMessageAndOffset(MessageAndOffset messageAndOffset) throws UnsupportedEncodingException { + ByteBuffer keyPayload = messageAndOffset.message() + .key(); + byte[] keyBytes = new byte[keyPayload.limit()]; + keyPayload.get(keyBytes); + String key = new String(keyBytes, "UTF-8"); + return key; + } + + private String getMessageFromMessageAndOffset( + MessageAndOffset messageAndOffset) throws UnsupportedEncodingException { + + ByteBuffer messagePayload = messageAndOffset.message() + .payload(); + + byte[] messageBytes = new byte[messagePayload.limit()]; + messagePayload.get(messageBytes); + String message = new String(messageBytes, "UTF-8"); + return message; + } + + private void handleFetchError(FetchResponse fetchResponse) { + short code = fetchResponse.errorCode(topic, partition); + TRACE.log( + TraceLevel.ERROR, + "Error fetching data from the Broker:" + + leadBroker.host() + " Reason: " + code); + + if (code == ErrorMapping.OffsetOutOfRangeCode()) { + // We asked for an invalid offset. This should never + // happen. + TRACE.log(TraceLevel.ERROR, + "Tried to request an invalid offset. Exiting."); + } + myConsumer.close(); + myConsumer = null; + leadBroker = findLeader(zkClient, topic, partition); + } + public void initialize(OperatorContext context) throws Exception { // super.initialize(context); @@ -135,24 +323,6 @@ public void run() { processThread.setDaemon(false); } - /** - * Notification that initialization is complete and all input and output - * ports are connected and ready to receive and submit tuples. - * - * @throws Exception - * Operator failure, will cause the enclosing PE to terminate. - */ - public synchronized void allPortsReady() throws Exception { - Logger.getLogger(this.getClass()).trace( - "Operator " + operContext.getName() - + " all ports are ready in PE: " - + operContext.getPE().getPEId() + " in Job: " - + operContext.getPE().getJobId()); - // Start a thread for producing tuples because operator - // implementations must not block and must return control to the caller. - processThread.start(); - } - /** * Submit new tuples to the output stream * @@ -171,7 +341,7 @@ private void produceTuples() throws Exception { TRACE.log( TraceLevel.TRACE, "Iteration through the while loop. ReadOffset: " - + Long.toString(readOffset)); + + readOffset); if (myConsumer == null) { myConsumer = new SimpleConsumer(leadBroker.host(), @@ -274,43 +444,26 @@ private void produceTuples() throws Exception { myConsumer.close(); } - private String getKeyFromMessageAndOffset(MessageAndOffset messageAndOffset) throws UnsupportedEncodingException { - ByteBuffer keyPayload = messageAndOffset.message() - .key(); - byte[] keyBytes = new byte[keyPayload.limit()]; - keyPayload.get(keyBytes); - String key = new String(keyBytes, "UTF-8"); - return key; - } + @Override + public void reset(Checkpoint checkpoint) throws Exception { + // System.out.println("Reset"); + TRACE.log(TraceLevel.INFO, + "Reset to checkpoint " + checkpoint.getSequenceId()); + resetReadOffset = checkpoint.getInputStream().readLong(); - private String getMessageFromMessageAndOffset( - MessageAndOffset messageAndOffset) throws UnsupportedEncodingException { - - ByteBuffer messagePayload = messageAndOffset.message() - .payload(); - - byte[] messageBytes = new byte[messagePayload.limit()]; - messagePayload.get(messageBytes); - String message = new String(messageBytes, "UTF-8"); - return message; + // System.out.println("Set readOffset to : " + readOffset); } - private void handleFetchError(FetchResponse fetchResponse) { - short code = fetchResponse.errorCode(topic, partition); - TRACE.log( - TraceLevel.ERROR, - "Error fetching data from the Broker:" - + leadBroker.host() + " Reason: " + code); + @Override + public void resetToInitialState() throws Exception { + // System.out.println("ResetToInitial"); + TRACE.log(TraceLevel.INFO, "Reset to initial state"); + } - if (code == ErrorMapping.OffsetOutOfRangeCode()) { - // We asked for an invalid offset. This should never - // happen. - TRACE.log(TraceLevel.ERROR, - "Tried to request an invalid offset. Exiting."); - } - myConsumer.close(); - myConsumer = null; - leadBroker = findLeader(zkClient, topic, partition); + @Override + public void retireCheckpoint(long id) throws Exception { + // System.out.println("Retire checkpoint"); + TRACE.log(TraceLevel.INFO, "Retire checkpoint"); } /** @@ -336,157 +489,4 @@ public synchronized void shutdown() throws Exception { // super.shutdown(); } - @Override - public void close() throws IOException { - TRACE.log(TraceLevel.INFO, "StateHandler close"); - } - - @Override - public void checkpoint(Checkpoint checkpoint) throws Exception { - // System.out.println("Checkpoint readOffset " + readOffset); - TRACE.log(TraceLevel.INFO, "Checkpoint " + checkpoint.getSequenceId()); - checkpoint.getOutputStream().writeLong(readOffset); - } - - @Override - public void drain() throws Exception { - // System.out.println("Drain..."); - TRACE.log(TraceLevel.INFO, "Drain..."); - } - - @Override - public void reset(Checkpoint checkpoint) throws Exception { - // System.out.println("Reset"); - TRACE.log(TraceLevel.INFO, - "Reset to checkpoint " + checkpoint.getSequenceId()); - resetReadOffset = checkpoint.getInputStream().readLong(); - - // System.out.println("Set readOffset to : " + readOffset); - } - - @Override - public void resetToInitialState() throws Exception { - // System.out.println("ResetToInitial"); - TRACE.log(TraceLevel.INFO, "Reset to initial state"); - } - - @Override - public void retireCheckpoint(long id) throws Exception { - // System.out.println("Retire checkpoint"); - TRACE.log(TraceLevel.INFO, "Retire checkpoint"); - } - - private ZkClient getInitializedZkClient() { - ZkClient localZkClient; - String zkConnect = finalProperties.getProperty("zookeeper.connect"); - int zkSessionTimeout = getIntegerProperty( - "zookeeper.session.timeout.ms", 400); - int zkConnectionTimeout = getIntegerProperty("zookeeper.sync.time.ms", - 200); - - TRACE.log(TraceLevel.INFO, - "Initializing ZooKeeper with values: zkConnect(" + zkConnect - + ") zkSessionTimeout(" + zkSessionTimeout - + ") zkConnectionTimeout(" + zkConnectionTimeout + ")"); - - try { - localZkClient = new ZkClient(zkConnect, zkSessionTimeout, - zkConnectionTimeout, ZKStringSerializer$.MODULE$); - } catch (Exception e) { - TRACE.log(TraceLevel.ERROR, - "Zookeeper client did not initialize correctly with exception: " - + e); - throw e; - } - - return localZkClient; - } - - private int getIntegerProperty(String propName, int defaultVal) { - int integerProp = defaultVal; - try { - integerProp = Integer.parseInt(finalProperties - .getProperty(propName)); - } catch (Exception e){ - e.printStackTrace(); - TRACE.log(TraceLevel.ERROR, "Property " + propName + "was not input as type int. Exception: " + e); - } - - return integerProp; - } - - private Broker findLeader(ZkClient a_zkClient, String a_topic, - int a_partition) { - int leaderID; - Broker a_leadBroker = null; - - try { - - leaderID = ZkUtils.getLeaderForPartition(zkClient, a_topic, - a_partition).get(); - a_leadBroker = ZkUtils.getBrokerInfo(zkClient, leaderID).get(); - - } catch (Exception e) { - - e.printStackTrace(); - System.out.println("Exception from ZkClient!"); - - } - - return a_leadBroker; - } - - private Broker findNewLeader(Broker oldBroker, ZkClient a_zkClient, - String a_topic, int a_partition) { - - for (int i = 0; i < 3; i++) { - boolean goSleep = false; - Broker newLeadBroker = findLeader(a_zkClient, a_topic, a_partition); - - if (newLeadBroker == null) { - goSleep = true; - } else if (newLeadBroker.host().equalsIgnoreCase(oldBroker.host()) - && i == 0) { - // if it's the first time, let's give zookeeper a chance to - // recover. - goSleep = true; - } else { - return newLeadBroker; - } - - if (goSleep) { - try { - Thread.sleep(1000); - } catch (InterruptedException ie) { - } - } - } - - return null; - } - - public static long getLastOffset(SimpleConsumer consumer, String topic, - int partition, long whichTime, String clientName) { - TopicAndPartition topicAndPartition = new TopicAndPartition(topic, - partition); - Map requestInfo = new HashMap(); - requestInfo.put(topicAndPartition, new PartitionOffsetRequestInfo( - whichTime, 1)); - kafka.javaapi.OffsetRequest request = new kafka.javaapi.OffsetRequest( - requestInfo, kafka.api.OffsetRequest.CurrentVersion(), - clientName); - OffsetResponse response = consumer.getOffsetsBefore(request); - - if (response.hasError()) { - TRACE.log(TraceLevel.ERROR, - "Error fetching data Offset Data the Broker. Reason: " - + response.errorCode(topic, partition)); - return 0; - } - long[] offsets = response.offsets(topic, partition); - TRACE.log(TraceLevel.TRACE, - "Retrieving offsets: " + Arrays.toString(offsets)); - return offsets[0]; - } - } From c40b347f013479fe340ff7876b511caadb883f02 Mon Sep 17 00:00:00 2001 From: Alex-Cook4 Date: Wed, 22 Jul 2015 17:41:55 -0500 Subject: [PATCH 06/14] More consistent region support and performance inlining --- .../streamsx/messaging/kafka/KafkaSource.java | 2 +- .../messaging/kafka/SimpleConsumerClient.java | 245 ++++++++++-------- 2 files changed, 139 insertions(+), 108 deletions(-) diff --git a/com.ibm.streamsx.messaging/impl/java/src/com/ibm/streamsx/messaging/kafka/KafkaSource.java b/com.ibm.streamsx.messaging/impl/java/src/com/ibm/streamsx/messaging/kafka/KafkaSource.java index 1ca715e..d8ea016 100644 --- a/com.ibm.streamsx.messaging/impl/java/src/com/ibm/streamsx/messaging/kafka/KafkaSource.java +++ b/com.ibm.streamsx.messaging/impl/java/src/com/ibm/streamsx/messaging/kafka/KafkaSource.java @@ -106,7 +106,7 @@ public void allPortsReady() throws Exception { trace.log(TraceLevel.INFO, "Initializing client"); if(a_partition >= 0){ trace.log(TraceLevel.INFO, "Using simple consumer client."); - simpleClient = new SimpleConsumerClient(topics.get(0), a_partition, topicAH, keyAH, messageAH, finalProperties, triggerCount); + simpleClient = new SimpleConsumerClient(topics.get(0), a_partition, keyAH, messageAH, finalProperties, triggerCount); simpleClient.initialize(getOperatorContext()); simpleClient.allPortsReady(); } else { diff --git a/com.ibm.streamsx.messaging/impl/java/src/com/ibm/streamsx/messaging/kafka/SimpleConsumerClient.java b/com.ibm.streamsx.messaging/impl/java/src/com/ibm/streamsx/messaging/kafka/SimpleConsumerClient.java index e42528a..7799df7 100644 --- a/com.ibm.streamsx.messaging/impl/java/src/com/ibm/streamsx/messaging/kafka/SimpleConsumerClient.java +++ b/com.ibm.streamsx.messaging/impl/java/src/com/ibm/streamsx/messaging/kafka/SimpleConsumerClient.java @@ -1,34 +1,37 @@ /* Generated by Streams Studio: May 26, 2015 1:20:31 PM EDT */ package com.ibm.streamsx.messaging.kafka; +import java.io.BufferedReader; +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileNotFoundException; +import java.io.FileReader; +import java.io.FileWriter; +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import java.nio.ByteBuffer; +import java.util.HashMap; +import java.util.Map; +import java.util.Properties; + import kafka.api.FetchRequest; import kafka.api.FetchRequestBuilder; import kafka.api.PartitionOffsetRequestInfo; import kafka.cluster.Broker; import kafka.common.ErrorMapping; import kafka.common.TopicAndPartition; -import kafka.javaapi.*; +import kafka.javaapi.FetchResponse; +import kafka.javaapi.OffsetResponse; import kafka.javaapi.consumer.SimpleConsumer; import kafka.message.MessageAndOffset; -import kafka.utils.ZkUtils; import kafka.utils.ZKStringSerializer$; -import org.I0Itec.zkclient.ZkClient; - - -import java.io.IOException; -import java.io.UnsupportedEncodingException; -import java.nio.ByteBuffer; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Properties; +import kafka.utils.ZkUtils; +import org.I0Itec.zkclient.ZkClient; import org.apache.log4j.Logger; import scala.actors.threadpool.Arrays; - import com.ibm.streams.operator.OperatorContext; import com.ibm.streams.operator.OutputTuple; import com.ibm.streams.operator.StreamingOutput; @@ -41,29 +44,7 @@ public class SimpleConsumerClient implements StateHandler { private static Logger TRACE = Logger.getLogger(SimpleConsumerClient.class .getCanonicalName()); - public static long getLastOffset(SimpleConsumer consumer, String topic, - int partition, long whichTime, String clientName) { - TopicAndPartition topicAndPartition = new TopicAndPartition(topic, - partition); - Map requestInfo = new HashMap(); - requestInfo.put(topicAndPartition, new PartitionOffsetRequestInfo( - whichTime, 1)); - kafka.javaapi.OffsetRequest request = new kafka.javaapi.OffsetRequest( - requestInfo, kafka.api.OffsetRequest.CurrentVersion(), - clientName); - OffsetResponse response = consumer.getOffsetsBefore(request); - - if (response.hasError()) { - TRACE.log(TraceLevel.ERROR, - "Error fetching data Offset Data the Broker. Reason: " - + response.errorCode(topic, partition)); - return 0; - } - long[] offsets = response.offsets(topic, partition); - TRACE.log(TraceLevel.TRACE, - "Retrieving offsets: " + Arrays.toString(offsets)); - return offsets[0]; - } + private Thread processThread; static final String OPER_NAME = "KafkaConsumer"; private ConsistentRegionContext crContext; @@ -71,25 +52,33 @@ public static long getLastOffset(SimpleConsumer consumer, String topic, private long triggerCount; private long triggerIteration = 0; private OperatorContext operContext; + private boolean inReset = false; // consumer variables public SimpleConsumer myConsumer = null; private ZkClient zkClient; private String topic; private Broker leadBroker; private long readOffset; - private long resetReadOffset = -1; private int partition; private int so_timeout; + private int bufferSize; + private int fetchSize; private String clientName; - protected Properties finalProperties = new Properties(); + private String charSet = "UTF-8"; + private AttributeHelper keyAH; + private AttributeHelper messageAH; + + private static final int DEFAULT_BUFFER_SIZE = 64 * 1024; + private static final int DEFAULT_FETCH_SIZE = 100 * 1024; + private static final int DEFAULT_SO_TIMEOUT = 30 * 1000; public SimpleConsumerClient(String a_topic, int a_partition, - AttributeHelper topicAH, AttributeHelper keyAH, + AttributeHelper keyAH, AttributeHelper messageAH, Properties props, long trigCnt) { this.topic = a_topic; - // this.keyAH = keyAH; - // this.messageAH = messageAH; + this.keyAH = keyAH; + this.messageAH = messageAH; this.partition = a_partition; this.finalProperties = props; this.triggerCount = trigCnt; @@ -181,7 +170,7 @@ private Broker findNewLeader(Broker oldBroker, ZkClient a_zkClient, return null; } - private ZkClient getInitializedZkClient() { + private ZkClient getInitializedZkClient() throws Exception{ ZkClient localZkClient; String zkConnect = finalProperties.getProperty("zookeeper.connect"); int zkSessionTimeout = getIntegerProperty( @@ -209,36 +198,44 @@ private ZkClient getInitializedZkClient() { private int getIntegerProperty(String propName, int defaultVal) { int integerProp = defaultVal; - try { - integerProp = Integer.parseInt(finalProperties - .getProperty(propName)); - } catch (Exception e){ - e.printStackTrace(); - TRACE.log(TraceLevel.ERROR, "Property " + propName + " was not input as type int. Exception: " + e); + String propVal = finalProperties + .getProperty(propName); + + if (propVal != null){ + try { + integerProp = Integer.parseInt(finalProperties + .getProperty(propName)); + } catch (Exception e){ + e.printStackTrace(); + TRACE.log(TraceLevel.ERROR, "Property " + propName + " was not input as type int. Exception: " + e); + } } return integerProp; } + + public static long getLastOffset(SimpleConsumer consumer, String topic, + int partition, long whichTime, String clientName) { + TopicAndPartition topicAndPartition = new TopicAndPartition(topic, + partition); + Map requestInfo = new HashMap(); + requestInfo.put(topicAndPartition, new PartitionOffsetRequestInfo( + whichTime, 1)); + kafka.javaapi.OffsetRequest request = new kafka.javaapi.OffsetRequest( + requestInfo, kafka.api.OffsetRequest.CurrentVersion(), + clientName); + OffsetResponse response = consumer.getOffsetsBefore(request); - private String getKeyFromMessageAndOffset(MessageAndOffset messageAndOffset) throws UnsupportedEncodingException { - ByteBuffer keyPayload = messageAndOffset.message() - .key(); - byte[] keyBytes = new byte[keyPayload.limit()]; - keyPayload.get(keyBytes); - String key = new String(keyBytes, "UTF-8"); - return key; - } - - private String getMessageFromMessageAndOffset( - MessageAndOffset messageAndOffset) throws UnsupportedEncodingException { - - ByteBuffer messagePayload = messageAndOffset.message() - .payload(); - - byte[] messageBytes = new byte[messagePayload.limit()]; - messagePayload.get(messageBytes); - String message = new String(messageBytes, "UTF-8"); - return message; + if (response.hasError()) { + TRACE.log(TraceLevel.ERROR, + "Error fetching data Offset Data the Broker. Reason: " + + response.errorCode(topic, partition)); + return 0; + } + long[] offsets = response.offsets(topic, partition); + TRACE.log(TraceLevel.TRACE, + "Retrieving offsets: " + Arrays.toString(offsets)); + return offsets[0]; } private void handleFetchError(FetchResponse fetchResponse) { @@ -261,7 +258,6 @@ private void handleFetchError(FetchResponse fetchResponse) { public void initialize(OperatorContext context) throws Exception { - // super.initialize(context); Logger.getLogger(this.getClass()).trace( "Operator " + context.getName() + " initializing in PE: " + context.getPE().getPEId() + " in Job: " @@ -276,8 +272,9 @@ public void initialize(OperatorContext context) throws Exception { TRACE.log(TraceLevel.TRACE, "Initializing consumer with clientName: " + clientName); - //default 30 seconds - so_timeout = Integer.parseInt(finalProperties.getProperty("socket.timeout.ms", "30000")); + so_timeout = getIntegerProperty("socket.timeout.ms", DEFAULT_SO_TIMEOUT); + bufferSize = getIntegerProperty("simpleConsumer.bufferSize.bytes", DEFAULT_BUFFER_SIZE); + fetchSize = getIntegerProperty("simpleConsumer.fetchSize.bytes", DEFAULT_FETCH_SIZE); zkClient = getInitializedZkClient(); leadBroker = findLeader(zkClient, topic, partition); @@ -288,14 +285,36 @@ public void initialize(OperatorContext context) throws Exception { shutdown(); return; } - + + if (!keyAH.isAvailable()){ + throw new Exception("Specified output attribute " + keyAH.getName() + " is not available."); + } + + if (!messageAH.isAvailable()){ + throw new Exception("Specified output attribute " + messageAH.getName() + " is not available."); + } + + myConsumer = new SimpleConsumer(leadBroker.host(), leadBroker.port(), - so_timeout, 64 * 1024, clientName); - TRACE.log(TraceLevel.TRACE, "Consumer initialization complete."); + so_timeout, bufferSize, clientName); + TRACE.log(TraceLevel.INFO, "Consumer initialization complete."); + TRACE.log(TraceLevel.INFO, + "Initializing SimpleConsumer with values: leadBroker(" + leadBroker.host() + ":" + leadBroker.port() + + ") socket timeout(" + so_timeout + + ") bufferSize(" + bufferSize + + ") clientName(" + clientName + ")"); readOffset = getLastOffset(myConsumer, topic, partition, kafka.api.OffsetRequest.LatestTime(), clientName); + try (BufferedWriter bw = new BufferedWriter(new FileWriter(new File("/tmp/initialOffset.txt")))){ + bw.write(Long.toString(readOffset)); + bw.close(); + }catch (FileNotFoundException ex) { + TRACE.log(TraceLevel.ERROR, "Failed to write out offset!"); + } + + /* * Create the thread for producing tuples. The thread is created at * initialize time but started. The thread will be started by @@ -308,6 +327,7 @@ public void run() { try { produceTuples(); } catch (Exception e) { + TRACE.log(TraceLevel.ERROR, "Operator error: " + e.getMessage() + "\n" + e.getStackTrace()); Logger.getLogger(this.getClass()) .error("Operator error", e); //$NON-NLS-1$ } @@ -329,7 +349,7 @@ public void run() { * @throws Exception * if an error occurs while submitting a tuple */ - private void produceTuples() throws Exception { + private void produceTuples() throws UnsupportedEncodingException { final StreamingOutput out = operContext .getStreamingOutputs().get(0); OutputTuple tuple = out.newTuple(); @@ -338,18 +358,15 @@ private void produceTuples() throws Exception { FetchResponse fetchResponse; while (!shutdown) { - TRACE.log( - TraceLevel.TRACE, - "Iteration through the while loop. ReadOffset: " - + readOffset); + if (myConsumer == null) { myConsumer = new SimpleConsumer(leadBroker.host(), - leadBroker.port(), so_timeout, 64 * 1024, clientName); + leadBroker.port(), so_timeout, bufferSize, clientName); } FetchRequest req = new FetchRequestBuilder().clientId(clientName) - .addFetch(topic, partition, readOffset, 100000).build(); + .addFetch(topic, partition, readOffset, fetchSize).build(); try { fetchResponse = myConsumer.fetch(req); @@ -374,34 +391,46 @@ private void produceTuples() throws Exception { numRead = 0; for (MessageAndOffset messageAndOffset : fetchResponse .messageSet(topic, partition)) { - currentOffset = messageAndOffset.offset(); - tuple = out.newTuple(); - - if (currentOffset < readOffset) { - TRACE.log(TraceLevel.ERROR, "Found an old offset: " - + currentOffset + " Expecting: " - + readOffset); - } - - String message = getMessageFromMessageAndOffset(messageAndOffset); - String key = getKeyFromMessageAndOffset(messageAndOffset); - - // Set attributes in tuple: - tuple.setString("message", message); - tuple.setString("key", key); - try { if (crContext != null) { crContext.acquirePermit(); } - // if there has been a reset, we NEED to get out of // this loop and do a new fetch request - if (resetReadOffset != -1) { - readOffset = resetReadOffset; - resetReadOffset = -1; + if (inReset) { + inReset = false; break; } + + currentOffset = messageAndOffset.offset(); + tuple = out.newTuple(); + + if (currentOffset < readOffset) { + TRACE.log(TraceLevel.ERROR, "Found an old offset: " + + currentOffset + " Expecting: " + + readOffset); + } + + + ByteBuffer messagePayload = messageAndOffset.message() + .payload(); + + byte[] messageBytes = new byte[messagePayload.limit()]; + messagePayload.get(messageBytes); + String message = new String(messageBytes, charSet); + + ByteBuffer keyPayload = messageAndOffset.message() + .key(); + byte[] keyBytes = new byte[keyPayload.limit()]; + keyPayload.get(keyBytes); + String key = new String(keyBytes, charSet); + + // Set attributes in tuple: + tuple.setString(messageAH.getName(), message); + tuple.setString(keyAH.getName(), key); + + + numRead++; // Submit tuple to output stream @@ -412,7 +441,6 @@ private void produceTuples() throws Exception { && crContext.isTriggerOperator()) { triggerIteration++; if (triggerIteration >= triggerCount) { - crContext.makeConsistent(); triggerIteration = 0; } @@ -449,8 +477,8 @@ public void reset(Checkpoint checkpoint) throws Exception { // System.out.println("Reset"); TRACE.log(TraceLevel.INFO, "Reset to checkpoint " + checkpoint.getSequenceId()); - resetReadOffset = checkpoint.getInputStream().readLong(); - + readOffset = checkpoint.getInputStream().readLong(); + inReset = true; // System.out.println("Set readOffset to : " + readOffset); } @@ -458,6 +486,11 @@ public void reset(Checkpoint checkpoint) throws Exception { public void resetToInitialState() throws Exception { // System.out.println("ResetToInitial"); TRACE.log(TraceLevel.INFO, "Reset to initial state"); + try(BufferedReader br = new BufferedReader(new FileReader("/tmp/initialOffset.txt"))) { + String line = br.readLine(); + readOffset = Long.parseLong(line); + TRACE.log(TraceLevel.INFO, "Retrieved initial state from file."); + } } @Override @@ -479,14 +512,12 @@ public synchronized void shutdown() throws Exception { processThread.interrupt(); processThread = null; } - // OperatorContext context = getOperatorContext(); + Logger.getLogger(this.getClass()).trace( "Operator " + operContext.getName() + " shutting down in PE: " + operContext.getPE().getPEId() + " in Job: " + operContext.getPE().getJobId()); - // Must call super.shutdown() - // super.shutdown(); } } From 94b8e116adf2a146f75b9beae9e04cc781d94d0b Mon Sep 17 00:00:00 2001 From: Alex-Cook4 Date: Thu, 23 Jul 2015 18:00:21 -0500 Subject: [PATCH 07/14] Protected Tracing and Parameters for leader search --- .../streamsx/messaging/kafka/KafkaSource.java | 18 +++- .../messaging/kafka/SimpleConsumerClient.java | 86 ++++++++++++------- 2 files changed, 70 insertions(+), 34 deletions(-) diff --git a/com.ibm.streamsx.messaging/impl/java/src/com/ibm/streamsx/messaging/kafka/KafkaSource.java b/com.ibm.streamsx.messaging/impl/java/src/com/ibm/streamsx/messaging/kafka/KafkaSource.java index d8ea016..0e7e2d2 100644 --- a/com.ibm.streamsx.messaging/impl/java/src/com/ibm/streamsx/messaging/kafka/KafkaSource.java +++ b/com.ibm.streamsx.messaging/impl/java/src/com/ibm/streamsx/messaging/kafka/KafkaSource.java @@ -32,7 +32,9 @@ public class KafkaSource extends KafkaBaseOper implements StateHandler{ static final String OPER_NAME = "KafkaConsumer"; private int threadsPerTopic = 1; private int a_partition = -1; - private int triggerCount = -1; + private int triggerCount = -1; + private int leaderConnectionRetries = 3; + private int connectionRetryInterval = 1000; private static Logger trace = Logger.getLogger(KafkaSource.class.getName()); //consistent region checks @@ -106,7 +108,7 @@ public void allPortsReady() throws Exception { trace.log(TraceLevel.INFO, "Initializing client"); if(a_partition >= 0){ trace.log(TraceLevel.INFO, "Using simple consumer client."); - simpleClient = new SimpleConsumerClient(topics.get(0), a_partition, keyAH, messageAH, finalProperties, triggerCount); + simpleClient = new SimpleConsumerClient(topics.get(0), a_partition, keyAH, messageAH, finalProperties, triggerCount, leaderConnectionRetries, connectionRetryInterval); simpleClient.initialize(getOperatorContext()); simpleClient.allPortsReady(); } else { @@ -137,6 +139,18 @@ public void setPartition(int value) { description="Number of messages between checkpointing for consistent region. This is only relevant to operator driven checkpointing.") public void setTriggerCount(int value) { this.triggerCount = value; + } + + @Parameter(name="leaderConnectionRetries", optional=true, + description="Number of attempts at finding a Kafka Broker before giving up. This is relevant when first looking for a broker, and in the case that a lead broker host goes down. Default is 3.") + public void setLeaderConnectionRetries(int value) { + this.leaderConnectionRetries = value; + } + + @Parameter(name="connectionRetryInterval", optional=true, + description="Interval between each attempt to find a lead Kafka Broker in milliseconds. Number of attempts is set by the leaderConnectionRetries parameter.") + public void setConnectionRetryInterval(int value) { + this.connectionRetryInterval = value; } public static final String DESC = diff --git a/com.ibm.streamsx.messaging/impl/java/src/com/ibm/streamsx/messaging/kafka/SimpleConsumerClient.java b/com.ibm.streamsx.messaging/impl/java/src/com/ibm/streamsx/messaging/kafka/SimpleConsumerClient.java index 7799df7..264a572 100644 --- a/com.ibm.streamsx.messaging/impl/java/src/com/ibm/streamsx/messaging/kafka/SimpleConsumerClient.java +++ b/com.ibm.streamsx.messaging/impl/java/src/com/ibm/streamsx/messaging/kafka/SimpleConsumerClient.java @@ -13,7 +13,7 @@ import java.util.HashMap; import java.util.Map; import java.util.Properties; - + import kafka.api.FetchRequest; import kafka.api.FetchRequestBuilder; import kafka.api.PartitionOffsetRequestInfo; @@ -68,6 +68,9 @@ public class SimpleConsumerClient implements StateHandler { private String charSet = "UTF-8"; private AttributeHelper keyAH; private AttributeHelper messageAH; + private int leaderConnectionRetries = 3; + private int connectionRetryInterval = 1000; + private static final int DEFAULT_BUFFER_SIZE = 64 * 1024; private static final int DEFAULT_FETCH_SIZE = 100 * 1024; @@ -75,13 +78,16 @@ public class SimpleConsumerClient implements StateHandler { public SimpleConsumerClient(String a_topic, int a_partition, AttributeHelper keyAH, - AttributeHelper messageAH, Properties props, long trigCnt) { + AttributeHelper messageAH, Properties props, long trigCnt, + int connectionRetries, int connectionRetryInterval) { this.topic = a_topic; this.keyAH = keyAH; this.messageAH = messageAH; this.partition = a_partition; this.finalProperties = props; this.triggerCount = trigCnt; + this.leaderConnectionRetries = connectionRetries; + this.connectionRetryInterval = connectionRetryInterval; } /** @@ -105,19 +111,22 @@ public synchronized void allPortsReady() throws Exception { @Override public void checkpoint(Checkpoint checkpoint) throws Exception { // System.out.println("Checkpoint readOffset " + readOffset); - TRACE.log(TraceLevel.INFO, "Checkpoint " + checkpoint.getSequenceId()); + if (TRACE.isInfoEnabled()) + TRACE.log(TraceLevel.INFO, "Checkpoint " + checkpoint.getSequenceId()); checkpoint.getOutputStream().writeLong(readOffset); } @Override public void close() throws IOException { - TRACE.log(TraceLevel.INFO, "StateHandler close"); + if (TRACE.isInfoEnabled()) + TRACE.log(TraceLevel.INFO, "StateHandler close"); } @Override public void drain() throws Exception { // System.out.println("Drain..."); - TRACE.log(TraceLevel.INFO, "Drain..."); + if (TRACE.isInfoEnabled()) + TRACE.log(TraceLevel.INFO, "Drain..."); } private Broker findLeader(ZkClient a_zkClient, String a_topic, @@ -135,6 +144,7 @@ private Broker findLeader(ZkClient a_zkClient, String a_topic, e.printStackTrace(); System.out.println("Exception from ZkClient!"); + TRACE.log(TraceLevel.ERROR, "Exception from ZkClient."); } @@ -142,15 +152,16 @@ private Broker findLeader(ZkClient a_zkClient, String a_topic, } private Broker findNewLeader(Broker oldBroker, ZkClient a_zkClient, - String a_topic, int a_partition) { + String a_topic, int a_partition, int numLeaderTries, int sleepBetweenTries) throws IOException{ - for (int i = 0; i < 3; i++) { + for (int i = 0; i < numLeaderTries; i++) { boolean goSleep = false; Broker newLeadBroker = findLeader(a_zkClient, a_topic, a_partition); if (newLeadBroker == null) { goSleep = true; - } else if (newLeadBroker.host().equalsIgnoreCase(oldBroker.host()) + } else if (oldBroker != null + && newLeadBroker.host().equalsIgnoreCase(oldBroker.host()) && i == 0) { // if it's the first time, let's give zookeeper a chance to // recover. @@ -161,13 +172,15 @@ private Broker findNewLeader(Broker oldBroker, ZkClient a_zkClient, if (goSleep) { try { - Thread.sleep(1000); + if(TRACE.isInfoEnabled()) + TRACE.log(TraceLevel.INFO, "Sleeping after attempt " + (i+1) + "."); + Thread.sleep(sleepBetweenTries); } catch (InterruptedException ie) { } } } - - return null; + + throw new IOException("Unable to find a new Kafka lead Broker after " + numLeaderTries + " attempts. Unable to proceed."); } private ZkClient getInitializedZkClient() throws Exception{ @@ -178,11 +191,13 @@ private ZkClient getInitializedZkClient() throws Exception{ int zkConnectionTimeout = getIntegerProperty("zookeeper.sync.time.ms", 200); - TRACE.log(TraceLevel.INFO, - "Initializing ZooKeeper with values: zkConnect(" + zkConnect - + ") zkSessionTimeout(" + zkSessionTimeout - + ") zkConnectionTimeout(" + zkConnectionTimeout + ")"); - + if (TRACE.isInfoEnabled()) + TRACE.log(TraceLevel.INFO, + "Initializing ZooKeeper with values: zkConnect(" + + zkConnect + ") zkSessionTimeout(" + + zkSessionTimeout + ") zkConnectionTimeout(" + + zkConnectionTimeout + ")"); + try { localZkClient = new ZkClient(zkConnect, zkSessionTimeout, zkConnectionTimeout, ZKStringSerializer$.MODULE$); @@ -238,7 +253,7 @@ public static long getLastOffset(SimpleConsumer consumer, String topic, return offsets[0]; } - private void handleFetchError(FetchResponse fetchResponse) { + private void handleFetchError(FetchResponse fetchResponse) throws IOException { short code = fetchResponse.errorCode(topic, partition); TRACE.log( TraceLevel.ERROR, @@ -253,7 +268,7 @@ private void handleFetchError(FetchResponse fetchResponse) { } myConsumer.close(); myConsumer = null; - leadBroker = findLeader(zkClient, topic, partition); + leadBroker = findNewLeader(null, zkClient, topic, partition, leaderConnectionRetries, connectionRetryInterval); } public void initialize(OperatorContext context) throws Exception { @@ -276,8 +291,9 @@ public void initialize(OperatorContext context) throws Exception { bufferSize = getIntegerProperty("simpleConsumer.bufferSize.bytes", DEFAULT_BUFFER_SIZE); fetchSize = getIntegerProperty("simpleConsumer.fetchSize.bytes", DEFAULT_FETCH_SIZE); + zkClient = getInitializedZkClient(); - leadBroker = findLeader(zkClient, topic, partition); + leadBroker = findNewLeader(null, zkClient, topic, partition, leaderConnectionRetries, connectionRetryInterval); if (leadBroker == null) { TRACE.log(TraceLevel.ERROR, @@ -297,12 +313,14 @@ public void initialize(OperatorContext context) throws Exception { myConsumer = new SimpleConsumer(leadBroker.host(), leadBroker.port(), so_timeout, bufferSize, clientName); - TRACE.log(TraceLevel.INFO, "Consumer initialization complete."); - TRACE.log(TraceLevel.INFO, - "Initializing SimpleConsumer with values: leadBroker(" + leadBroker.host() + ":" + leadBroker.port() - + ") socket timeout(" + so_timeout - + ") bufferSize(" + bufferSize - + ") clientName(" + clientName + ")"); + if (TRACE.isInfoEnabled()) + TRACE.log(TraceLevel.INFO, "Consumer initialization complete."); + if (TRACE.isInfoEnabled()) + TRACE.log(TraceLevel.INFO, + "Initializing SimpleConsumer with values: leadBroker(" + + leadBroker.host() + ":" + leadBroker.port() + + ") socket timeout(" + so_timeout + + ") bufferSize(" + bufferSize + ") clientName(" + clientName + ")"); readOffset = getLastOffset(myConsumer, topic, partition, kafka.api.OffsetRequest.LatestTime(), clientName); @@ -349,7 +367,7 @@ public void run() { * @throws Exception * if an error occurs while submitting a tuple */ - private void produceTuples() throws UnsupportedEncodingException { + private void produceTuples() throws UnsupportedEncodingException,IOException { final StreamingOutput out = operContext .getStreamingOutputs().get(0); OutputTuple tuple = out.newTuple(); @@ -379,7 +397,7 @@ private void produceTuples() throws UnsupportedEncodingException { myConsumer = null; fetchResponse = null; leadBroker = findNewLeader(leadBroker, zkClient, topic, - partition); + partition, leaderConnectionRetries, connectionRetryInterval); } if (fetchResponse != null) { @@ -475,8 +493,9 @@ private void produceTuples() throws UnsupportedEncodingException { @Override public void reset(Checkpoint checkpoint) throws Exception { // System.out.println("Reset"); - TRACE.log(TraceLevel.INFO, - "Reset to checkpoint " + checkpoint.getSequenceId()); + if (TRACE.isInfoEnabled()) + TRACE.log(TraceLevel.INFO, + "Reset to checkpoint " + checkpoint.getSequenceId()); readOffset = checkpoint.getInputStream().readLong(); inReset = true; // System.out.println("Set readOffset to : " + readOffset); @@ -485,18 +504,21 @@ public void reset(Checkpoint checkpoint) throws Exception { @Override public void resetToInitialState() throws Exception { // System.out.println("ResetToInitial"); - TRACE.log(TraceLevel.INFO, "Reset to initial state"); + if (TRACE.isInfoEnabled()) + TRACE.log(TraceLevel.INFO, "Reset to initial state"); try(BufferedReader br = new BufferedReader(new FileReader("/tmp/initialOffset.txt"))) { String line = br.readLine(); readOffset = Long.parseLong(line); - TRACE.log(TraceLevel.INFO, "Retrieved initial state from file."); + if (TRACE.isInfoEnabled()) + TRACE.log(TraceLevel.INFO, "Retrieved initial state from file."); } } @Override public void retireCheckpoint(long id) throws Exception { // System.out.println("Retire checkpoint"); - TRACE.log(TraceLevel.INFO, "Retire checkpoint"); + if (TRACE.isInfoEnabled()) + TRACE.log(TraceLevel.INFO, "Retire checkpoint"); } /** From b19854044016630d379b601f531b6e1bd559a933 Mon Sep 17 00:00:00 2001 From: Alex-Cook4 Date: Thu, 23 Jul 2015 18:13:35 -0500 Subject: [PATCH 08/14] Take out file read/write --- .../messaging/kafka/SimpleConsumerClient.java | 25 ++++++++++--------- 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/com.ibm.streamsx.messaging/impl/java/src/com/ibm/streamsx/messaging/kafka/SimpleConsumerClient.java b/com.ibm.streamsx.messaging/impl/java/src/com/ibm/streamsx/messaging/kafka/SimpleConsumerClient.java index 264a572..10b208c 100644 --- a/com.ibm.streamsx.messaging/impl/java/src/com/ibm/streamsx/messaging/kafka/SimpleConsumerClient.java +++ b/com.ibm.streamsx.messaging/impl/java/src/com/ibm/streamsx/messaging/kafka/SimpleConsumerClient.java @@ -325,12 +325,12 @@ public void initialize(OperatorContext context) throws Exception { readOffset = getLastOffset(myConsumer, topic, partition, kafka.api.OffsetRequest.LatestTime(), clientName); - try (BufferedWriter bw = new BufferedWriter(new FileWriter(new File("/tmp/initialOffset.txt")))){ - bw.write(Long.toString(readOffset)); - bw.close(); - }catch (FileNotFoundException ex) { - TRACE.log(TraceLevel.ERROR, "Failed to write out offset!"); - } +// try (BufferedWriter bw = new BufferedWriter(new FileWriter(new File("/tmp/initialOffset.txt")))){ +// bw.write(Long.toString(readOffset)); +// bw.close(); +// }catch (FileNotFoundException ex) { +// TRACE.log(TraceLevel.ERROR, "Failed to write out offset!"); +// } /* @@ -506,12 +506,13 @@ public void resetToInitialState() throws Exception { // System.out.println("ResetToInitial"); if (TRACE.isInfoEnabled()) TRACE.log(TraceLevel.INFO, "Reset to initial state"); - try(BufferedReader br = new BufferedReader(new FileReader("/tmp/initialOffset.txt"))) { - String line = br.readLine(); - readOffset = Long.parseLong(line); - if (TRACE.isInfoEnabled()) - TRACE.log(TraceLevel.INFO, "Retrieved initial state from file."); - } +// try { +// BufferedReader br = new BufferedReader(new FileReader("/tmp/initialOffset.txt")); +// String line = br.readLine(); +// readOffset = Long.parseLong(line); +// if (TRACE.isInfoEnabled()) +// TRACE.log(TraceLevel.INFO, "Retrieved initial state from file."); +// } } @Override From 2c0aeed78d520ea3d4845f550ecb923ed5bc2d32 Mon Sep 17 00:00:00 2001 From: Alex-Cook4 Date: Sun, 26 Jul 2015 21:00:15 -0500 Subject: [PATCH 09/14] Change for resetToInitialState --- .../messaging/kafka/SimpleConsumerClient.java | 87 +++++-------------- 1 file changed, 24 insertions(+), 63 deletions(-) diff --git a/com.ibm.streamsx.messaging/impl/java/src/com/ibm/streamsx/messaging/kafka/SimpleConsumerClient.java b/com.ibm.streamsx.messaging/impl/java/src/com/ibm/streamsx/messaging/kafka/SimpleConsumerClient.java index 10b208c..dd1f2a7 100644 --- a/com.ibm.streamsx.messaging/impl/java/src/com/ibm/streamsx/messaging/kafka/SimpleConsumerClient.java +++ b/com.ibm.streamsx.messaging/impl/java/src/com/ibm/streamsx/messaging/kafka/SimpleConsumerClient.java @@ -1,12 +1,6 @@ /* Generated by Streams Studio: May 26, 2015 1:20:31 PM EDT */ package com.ibm.streamsx.messaging.kafka; -import java.io.BufferedReader; -import java.io.BufferedWriter; -import java.io.File; -import java.io.FileNotFoundException; -import java.io.FileReader; -import java.io.FileWriter; import java.io.IOException; import java.io.UnsupportedEncodingException; import java.nio.ByteBuffer; @@ -70,7 +64,7 @@ public class SimpleConsumerClient implements StateHandler { private AttributeHelper messageAH; private int leaderConnectionRetries = 3; private int connectionRetryInterval = 1000; - + final long initialOffsetTime = kafka.api.OffsetRequest.LatestTime(); private static final int DEFAULT_BUFFER_SIZE = 64 * 1024; private static final int DEFAULT_FETCH_SIZE = 100 * 1024; @@ -110,9 +104,6 @@ public synchronized void allPortsReady() throws Exception { @Override public void checkpoint(Checkpoint checkpoint) throws Exception { - // System.out.println("Checkpoint readOffset " + readOffset); - if (TRACE.isInfoEnabled()) - TRACE.log(TraceLevel.INFO, "Checkpoint " + checkpoint.getSequenceId()); checkpoint.getOutputStream().writeLong(readOffset); } @@ -124,9 +115,6 @@ public void close() throws IOException { @Override public void drain() throws Exception { - // System.out.println("Drain..."); - if (TRACE.isInfoEnabled()) - TRACE.log(TraceLevel.INFO, "Drain..."); } private Broker findLeader(ZkClient a_zkClient, String a_topic, @@ -141,11 +129,7 @@ private Broker findLeader(ZkClient a_zkClient, String a_topic, a_leadBroker = ZkUtils.getBrokerInfo(zkClient, leaderID).get(); } catch (Exception e) { - - e.printStackTrace(); - System.out.println("Exception from ZkClient!"); - TRACE.log(TraceLevel.ERROR, "Exception from ZkClient."); - + TRACE.log(TraceLevel.ERROR, "Exception from ZkClient. ", e); } return a_leadBroker; @@ -221,11 +205,11 @@ private int getIntegerProperty(String propName, int defaultVal) { integerProp = Integer.parseInt(finalProperties .getProperty(propName)); } catch (Exception e){ - e.printStackTrace(); - TRACE.log(TraceLevel.ERROR, "Property " + propName + " was not input as type int. Exception: " + e); + + TRACE.log(TraceLevel.ERROR, "Property " + propName + " was not input as type int.", e); } } - + TRACE.log(TraceLevel.ERROR, "Property " + propName + " has a final value of " + integerProp); return integerProp; } @@ -313,24 +297,18 @@ public void initialize(OperatorContext context) throws Exception { myConsumer = new SimpleConsumer(leadBroker.host(), leadBroker.port(), so_timeout, bufferSize, clientName); - if (TRACE.isInfoEnabled()) - TRACE.log(TraceLevel.INFO, "Consumer initialization complete."); + if (TRACE.isInfoEnabled()) TRACE.log(TraceLevel.INFO, "Initializing SimpleConsumer with values: leadBroker(" + leadBroker.host() + ":" + leadBroker.port() + ") socket timeout(" + so_timeout + ") bufferSize(" + bufferSize + ") clientName(" + clientName + ")"); - + + + readOffset = getLastOffset(myConsumer, topic, partition, - kafka.api.OffsetRequest.LatestTime(), clientName); - -// try (BufferedWriter bw = new BufferedWriter(new FileWriter(new File("/tmp/initialOffset.txt")))){ -// bw.write(Long.toString(readOffset)); -// bw.close(); -// }catch (FileNotFoundException ex) { -// TRACE.log(TraceLevel.ERROR, "Failed to write out offset!"); -// } + initialOffsetTime, clientName); /* @@ -389,7 +367,6 @@ private void produceTuples() throws UnsupportedEncodingException,IOException { try { fetchResponse = myConsumer.fetch(req); } catch (Exception e) { - // System.out.println("Catching exception " + e); TRACE.log(TraceLevel.ERROR, "Fetch error. Lead server cannot be contacted. Exception: " + e.getStackTrace()); @@ -419,36 +396,33 @@ private void produceTuples() throws UnsupportedEncodingException,IOException { inReset = false; break; } - + currentOffset = messageAndOffset.offset(); tuple = out.newTuple(); - + if (currentOffset < readOffset) { - TRACE.log(TraceLevel.ERROR, "Found an old offset: " - + currentOffset + " Expecting: " - + readOffset); + TRACE.log(TraceLevel.ERROR, + "Found an old offset: " + currentOffset + + " Expecting: " + readOffset); } - - - ByteBuffer messagePayload = messageAndOffset.message() - .payload(); - - byte[] messageBytes = new byte[messagePayload.limit()]; + + ByteBuffer messagePayload = messageAndOffset + .message().payload(); + + byte[] messageBytes = new byte[messagePayload + .limit()]; messagePayload.get(messageBytes); String message = new String(messageBytes, charSet); - + ByteBuffer keyPayload = messageAndOffset.message() .key(); byte[] keyBytes = new byte[keyPayload.limit()]; keyPayload.get(keyBytes); - String key = new String(keyBytes, charSet); - + String key = new String(keyBytes, charSet); + // Set attributes in tuple: tuple.setString(messageAH.getName(), message); tuple.setString(keyAH.getName(), key); - - - numRead++; // Submit tuple to output stream @@ -492,34 +466,21 @@ private void produceTuples() throws UnsupportedEncodingException,IOException { @Override public void reset(Checkpoint checkpoint) throws Exception { - // System.out.println("Reset"); if (TRACE.isInfoEnabled()) TRACE.log(TraceLevel.INFO, "Reset to checkpoint " + checkpoint.getSequenceId()); readOffset = checkpoint.getInputStream().readLong(); inReset = true; - // System.out.println("Set readOffset to : " + readOffset); } @Override public void resetToInitialState() throws Exception { - // System.out.println("ResetToInitial"); if (TRACE.isInfoEnabled()) TRACE.log(TraceLevel.INFO, "Reset to initial state"); -// try { -// BufferedReader br = new BufferedReader(new FileReader("/tmp/initialOffset.txt")); -// String line = br.readLine(); -// readOffset = Long.parseLong(line); -// if (TRACE.isInfoEnabled()) -// TRACE.log(TraceLevel.INFO, "Retrieved initial state from file."); -// } } @Override public void retireCheckpoint(long id) throws Exception { - // System.out.println("Retire checkpoint"); - if (TRACE.isInfoEnabled()) - TRACE.log(TraceLevel.INFO, "Retire checkpoint"); } /** From fb454ff86561ae67e9cc6abb6e5d8e13925f3b0b Mon Sep 17 00:00:00 2001 From: Alex-Cook4 Date: Tue, 4 Aug 2015 13:58:26 -0500 Subject: [PATCH 10/14] Up version number --- com.ibm.streamsx.messaging/info.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/com.ibm.streamsx.messaging/info.xml b/com.ibm.streamsx.messaging/info.xml index 56cf817..34ae194 100644 --- a/com.ibm.streamsx.messaging/info.xml +++ b/com.ibm.streamsx.messaging/info.xml @@ -680,7 +680,7 @@ The <attribute> element has three possible attributes: * composite types * xml - 2.0.1 + 2.0.2 4.0.0.0 From deb90acbd20ac0f1e67f59b65ef722587cfbd9b5 Mon Sep 17 00:00:00 2001 From: Alex-Cook4 Date: Tue, 4 Aug 2015 14:02:23 -0500 Subject: [PATCH 11/14] Revert "Up version number" This reverts commit fb454ff86561ae67e9cc6abb6e5d8e13925f3b0b. --- com.ibm.streamsx.messaging/info.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/com.ibm.streamsx.messaging/info.xml b/com.ibm.streamsx.messaging/info.xml index 34ae194..56cf817 100644 --- a/com.ibm.streamsx.messaging/info.xml +++ b/com.ibm.streamsx.messaging/info.xml @@ -680,7 +680,7 @@ The <attribute> element has three possible attributes: * composite types * xml - 2.0.2 + 2.0.1 4.0.0.0 From 318bb38846cdd955e111855953913e64ca87fd44 Mon Sep 17 00:00:00 2001 From: Alex-Cook4 Date: Mon, 10 Aug 2015 18:41:43 -0500 Subject: [PATCH 12/14] Adding Consistent Region Samples --- .../.classpath | 12 +++ .../.project | 29 +++++++ .../Makefile | 31 ++++++++ .../application/.namespace | 0 .../ConsistentRegionConsumerParallel.spl | 77 +++++++++++++++++++ .../etc/consumer.properties | 6 ++ .../etc/producer.properties | 3 + .../info.xml | 15 ++++ .../.classpath | 12 +++ .../.project | 29 +++++++ .../Makefile | 31 ++++++++ .../application/.namespace | 0 .../ConsistentRegionConsumerSimple.spl | 72 +++++++++++++++++ .../etc/consumer.properties | 6 ++ .../etc/producer.properties | 3 + .../info.xml | 15 ++++ 16 files changed, 341 insertions(+) create mode 100644 samples/KafkaConsistentRegionConsumerParallel/.classpath create mode 100644 samples/KafkaConsistentRegionConsumerParallel/.project create mode 100644 samples/KafkaConsistentRegionConsumerParallel/Makefile create mode 100644 samples/KafkaConsistentRegionConsumerParallel/application/.namespace create mode 100644 samples/KafkaConsistentRegionConsumerParallel/application/ConsistentRegionConsumerParallel.spl create mode 100644 samples/KafkaConsistentRegionConsumerParallel/etc/consumer.properties create mode 100644 samples/KafkaConsistentRegionConsumerParallel/etc/producer.properties create mode 100644 samples/KafkaConsistentRegionConsumerParallel/info.xml create mode 100644 samples/KafkaConsistentRegionConsumerSimple/.classpath create mode 100644 samples/KafkaConsistentRegionConsumerSimple/.project create mode 100644 samples/KafkaConsistentRegionConsumerSimple/Makefile create mode 100644 samples/KafkaConsistentRegionConsumerSimple/application/.namespace create mode 100644 samples/KafkaConsistentRegionConsumerSimple/application/ConsistentRegionConsumerSimple.spl create mode 100644 samples/KafkaConsistentRegionConsumerSimple/etc/consumer.properties create mode 100644 samples/KafkaConsistentRegionConsumerSimple/etc/producer.properties create mode 100644 samples/KafkaConsistentRegionConsumerSimple/info.xml diff --git a/samples/KafkaConsistentRegionConsumerParallel/.classpath b/samples/KafkaConsistentRegionConsumerParallel/.classpath new file mode 100644 index 0000000..cbb0e4c --- /dev/null +++ b/samples/KafkaConsistentRegionConsumerParallel/.classpath @@ -0,0 +1,12 @@ + + + + + + + + + + + + diff --git a/samples/KafkaConsistentRegionConsumerParallel/.project b/samples/KafkaConsistentRegionConsumerParallel/.project new file mode 100644 index 0000000..a7fbe31 --- /dev/null +++ b/samples/KafkaConsistentRegionConsumerParallel/.project @@ -0,0 +1,29 @@ + + + KafkaConsistentRegionConsumerParallel + + + + + + org.eclipse.jdt.core.javabuilder + + + + + com.ibm.streams.studio.splproject.builder.SPLProjectBuilder + + + + + org.eclipse.xtext.ui.shared.xtextBuilder + + + + + + com.ibm.streams.studio.splproject.SPLProjectNature + org.eclipse.xtext.ui.shared.xtextNature + org.eclipse.jdt.core.javanature + + diff --git a/samples/KafkaConsistentRegionConsumerParallel/Makefile b/samples/KafkaConsistentRegionConsumerParallel/Makefile new file mode 100644 index 0000000..0624e30 --- /dev/null +++ b/samples/KafkaConsistentRegionConsumerParallel/Makefile @@ -0,0 +1,31 @@ +# Copyright (C) 2015, International Business Machines Corporation +# All Rights Reserved + +.PHONY: all clean + +TOOLKIT_NAME=com.ibm.streamsx.messaging +STREAMS_MESSAGING_TOOLKIT ?= $(shell ([ -e ../../$(TOOLKIT_NAME)/toolkit.xml ] && echo ../../$(TOOLKIT_NAME)) ||\ + ([ -e "../$(TOOLKIT_NAME)" ] && echo ../$(TOOLKIT_NAME)) ||\ + echo $(STREAMS_INSTALL)/toolkits/$(TOOLKIT_NAME)) + + +SPLC_FLAGS ?= -a --data-directory data +SPLC = $(STREAMS_INSTALL)/bin/sc + +SPL_CMD_ARGS ?= -t $(STREAMS_MESSAGING_TOOLKIT) +SPL_MAIN_COMPOSITE = application::ConsistentRegionConsumerParallel + +all: distributed + +data: + mkdir data + +standalone: data + $(SPLC) $(SPLC_FLAGS) -T -M $(SPL_MAIN_COMPOSITE) $(SPL_CMD_ARGS) + +distributed: data + $(SPLC) $(SPLC_FLAGS) -M $(SPL_MAIN_COMPOSITE) $(SPL_CMD_ARGS) + +clean: + $(SPLC) $(SPLC_FLAGS) -C -M $(SPL_MAIN_COMPOSITE) + rm data/*.out \ No newline at end of file diff --git a/samples/KafkaConsistentRegionConsumerParallel/application/.namespace b/samples/KafkaConsistentRegionConsumerParallel/application/.namespace new file mode 100644 index 0000000..e69de29 diff --git a/samples/KafkaConsistentRegionConsumerParallel/application/ConsistentRegionConsumerParallel.spl b/samples/KafkaConsistentRegionConsumerParallel/application/ConsistentRegionConsumerParallel.spl new file mode 100644 index 0000000..af88690 --- /dev/null +++ b/samples/KafkaConsistentRegionConsumerParallel/application/ConsistentRegionConsumerParallel.spl @@ -0,0 +1,77 @@ +namespace application ; + +use com.ibm.streamsx.messaging.kafka::* ; +/** + * Read from a three-partition Kafka topic using the KafkaConsumer operator + * in a consistent region (guaranteed tuple processing). This sample contains + * two consistent regions. The first includes the beacon and the KafkaProducer, + * the second includes the three parallel KafkaConsumers and the MessagePrinter. + * Kafka only guarantees ordering of tuples within a single partition. This application + * provided the same guarantee, but since we are reading from three separate partitions, + * we can lose order. Depending on the key/message, order can be recovered after consuming + * from a Kafka Server. + * + * Make sure you have created your topic before launching: + * bin/kafka-topics.sh --create --zookeeper :2181 --partitions 3 --topic myParallelTopic + * + * Edit the consumer.properties and producer.properties files found in the etc directory to include + * your Kafka properties. + * + * Build using Studio or the provided Makefile. + * + * Check results by looking at messagesReceived.out in the data directory. + * + * Consistent Region does not support Standalone mode, so this sample is only interesting in + * Distributed mode. + * + * + */ +composite ConsistentRegionConsumerParallel +{ + graph + //generate data to be written to a kafka server + @consistent(trigger = operatorDriven) + stream OutputStream = Beacon() + { + param + period : 0.25 ; + initDelay : 4.0 ; + triggerCount : 20u ; + output + OutputStream : topic = "myParallelTopic", message =(rstring) + IterationCount(), key =(rstring)(int32)(random() * 10.0) ; + } + + //Write to Kafka Server + () as KafkaSinkOp = KafkaProducer(OutputStream) + { + param + propertiesFile : "etc/producer.properties" ; + } + + //Read in from a kafka server and start consistent region + @parallel(width = 3) @consistent(trigger = periodic, period = 5.0) + stream KafkaConsumerOut = KafkaConsumer() + { + param + propertiesFile : "etc/consumer.properties" ; + topic : "myParallelTopic" ; + partition : getChannel() ; + } + + + //Print out data to a file + () as MessagePrinter = FileSink(KafkaConsumerOut) + { + param + file : "messagesReceived.out" ; + flush : 1u ; + format : csv ; + } + + () as JCP = JobControlPlane() + { + } + + } + diff --git a/samples/KafkaConsistentRegionConsumerParallel/etc/consumer.properties b/samples/KafkaConsistentRegionConsumerParallel/etc/consumer.properties new file mode 100644 index 0000000..fee372e --- /dev/null +++ b/samples/KafkaConsistentRegionConsumerParallel/etc/consumer.properties @@ -0,0 +1,6 @@ +zookeeper.connect=zk.host.1:2181,zk.host.2:2181,zk.host.3:2181 +serializer.class=kafka.serializer.StringEncoder +group.id=mygroup +zookeeper.session.timeout.ms=4000 +zookeeper.sync.time.ms=2000 +auto.commit.interval.ms=1000 \ No newline at end of file diff --git a/samples/KafkaConsistentRegionConsumerParallel/etc/producer.properties b/samples/KafkaConsistentRegionConsumerParallel/etc/producer.properties new file mode 100644 index 0000000..ebf7396 --- /dev/null +++ b/samples/KafkaConsistentRegionConsumerParallel/etc/producer.properties @@ -0,0 +1,3 @@ +metadata.broker.list=zk.host.1:2181,zk.host.2:2181,zk.host.3:2181 +serializer.class=kafka.serializer.StringEncoder +request.required.acks=1 \ No newline at end of file diff --git a/samples/KafkaConsistentRegionConsumerParallel/info.xml b/samples/KafkaConsistentRegionConsumerParallel/info.xml new file mode 100644 index 0000000..5cc8e9b --- /dev/null +++ b/samples/KafkaConsistentRegionConsumerParallel/info.xml @@ -0,0 +1,15 @@ + + + + ConsistentRegionConsumerParallel + + 1.0.0 + 4.0.1.0 + + + + com.ibm.streamsx.messaging + [2.0.2,3.0.0) + + + \ No newline at end of file diff --git a/samples/KafkaConsistentRegionConsumerSimple/.classpath b/samples/KafkaConsistentRegionConsumerSimple/.classpath new file mode 100644 index 0000000..cbb0e4c --- /dev/null +++ b/samples/KafkaConsistentRegionConsumerSimple/.classpath @@ -0,0 +1,12 @@ + + + + + + + + + + + + diff --git a/samples/KafkaConsistentRegionConsumerSimple/.project b/samples/KafkaConsistentRegionConsumerSimple/.project new file mode 100644 index 0000000..57bb480 --- /dev/null +++ b/samples/KafkaConsistentRegionConsumerSimple/.project @@ -0,0 +1,29 @@ + + + KafkaConsistentRegionConsumerSimple + + + + + + org.eclipse.jdt.core.javabuilder + + + + + com.ibm.streams.studio.splproject.builder.SPLProjectBuilder + + + + + org.eclipse.xtext.ui.shared.xtextBuilder + + + + + + com.ibm.streams.studio.splproject.SPLProjectNature + org.eclipse.xtext.ui.shared.xtextNature + org.eclipse.jdt.core.javanature + + diff --git a/samples/KafkaConsistentRegionConsumerSimple/Makefile b/samples/KafkaConsistentRegionConsumerSimple/Makefile new file mode 100644 index 0000000..8e5b0d2 --- /dev/null +++ b/samples/KafkaConsistentRegionConsumerSimple/Makefile @@ -0,0 +1,31 @@ +# Copyright (C) 2015, International Business Machines Corporation +# All Rights Reserved + +.PHONY: all clean + +TOOLKIT_NAME=com.ibm.streamsx.messaging +STREAMS_MESSAGING_TOOLKIT ?= $(shell ([ -e ../../$(TOOLKIT_NAME)/toolkit.xml ] && echo ../../$(TOOLKIT_NAME)) ||\ + ([ -e "../$(TOOLKIT_NAME)" ] && echo ../$(TOOLKIT_NAME)) ||\ + echo $(STREAMS_INSTALL)/toolkits/$(TOOLKIT_NAME)) + + +SPLC_FLAGS ?= -a --data-directory data +SPLC = $(STREAMS_INSTALL)/bin/sc + +SPL_CMD_ARGS ?= -t $(STREAMS_MESSAGING_TOOLKIT) +SPL_MAIN_COMPOSITE = application::ConsistentRegionConsumerSimple + +all: distributed + +data: + mkdir data + +standalone: data + $(SPLC) $(SPLC_FLAGS) -T -M $(SPL_MAIN_COMPOSITE) $(SPL_CMD_ARGS) + +distributed: data + $(SPLC) $(SPLC_FLAGS) -M $(SPL_MAIN_COMPOSITE) $(SPL_CMD_ARGS) + +clean: + $(SPLC) $(SPLC_FLAGS) -C -M $(SPL_MAIN_COMPOSITE) + rm data/*.out \ No newline at end of file diff --git a/samples/KafkaConsistentRegionConsumerSimple/application/.namespace b/samples/KafkaConsistentRegionConsumerSimple/application/.namespace new file mode 100644 index 0000000..e69de29 diff --git a/samples/KafkaConsistentRegionConsumerSimple/application/ConsistentRegionConsumerSimple.spl b/samples/KafkaConsistentRegionConsumerSimple/application/ConsistentRegionConsumerSimple.spl new file mode 100644 index 0000000..62938a6 --- /dev/null +++ b/samples/KafkaConsistentRegionConsumerSimple/application/ConsistentRegionConsumerSimple.spl @@ -0,0 +1,72 @@ +namespace application ; + +use com.ibm.streamsx.messaging.kafka::* ; +/** + * Read from a single-partition Kafka topic using the KafkaConsumer operator + * in a consistent region (guaranteed tuple processing). This sample contains + * two consistent regions. The first includes the beacon and the KafkaProducer, + * the second includes the KafkaConsumer and the MessagePrinter. + * + * Make sure you have created your topic before launching: + * bin/kafka-topics.sh --create --zookeeper :2181 --partitions 1 --topic mySimpleTopic + * + * Edit the consumer.properties and producer.properties files found in the etc directory to include + * your Kafka properties. + * + * Build using Studio or the provided Makefile. + * + * Check results by looking at messagesReceived.out in the data directory. + * + * Consistent Region does not support Standalone mode, so this sample is only interesting in + * Distributed mode. + * + * + */ +composite ConsistentRegionConsumerSimple +{ + graph + //generate data to be written to a kafka server + @consistent(trigger = operatorDriven) + stream OutputStream = Beacon() + { + param + period : 0.25 ; + initDelay : 4.0 ; + triggerCount : 20u ; + output + OutputStream : topic = "mySimpleTopic", message =(rstring) + IterationCount(), key =(rstring)(int32)(random() * 10.0) ; + } + + //Write to Kafka Server + () as KafkaSinkOp = KafkaProducer(OutputStream) + { + param + propertiesFile : "etc/producer.properties" ; + } + + //Read in from a kafka server and start consistent region + @consistent(trigger = operatorDriven) stream + KafkaConsumerOut = KafkaConsumer() + { + param + propertiesFile : "etc/consumer.properties" ; + topic : "mySimpleTopic" ; + partition : 0 ; + triggerCount : 20 ; + } + + //Print out data to a file + () as MessagePrinter = FileSink(KafkaConsumerOut) + { + param + file : "messagesReceived.out" ; + flush : 1u ; + format : csv ; + } + + () as JCP = JobControlPlane() + { + } + + } diff --git a/samples/KafkaConsistentRegionConsumerSimple/etc/consumer.properties b/samples/KafkaConsistentRegionConsumerSimple/etc/consumer.properties new file mode 100644 index 0000000..fee372e --- /dev/null +++ b/samples/KafkaConsistentRegionConsumerSimple/etc/consumer.properties @@ -0,0 +1,6 @@ +zookeeper.connect=zk.host.1:2181,zk.host.2:2181,zk.host.3:2181 +serializer.class=kafka.serializer.StringEncoder +group.id=mygroup +zookeeper.session.timeout.ms=4000 +zookeeper.sync.time.ms=2000 +auto.commit.interval.ms=1000 \ No newline at end of file diff --git a/samples/KafkaConsistentRegionConsumerSimple/etc/producer.properties b/samples/KafkaConsistentRegionConsumerSimple/etc/producer.properties new file mode 100644 index 0000000..ebf7396 --- /dev/null +++ b/samples/KafkaConsistentRegionConsumerSimple/etc/producer.properties @@ -0,0 +1,3 @@ +metadata.broker.list=zk.host.1:2181,zk.host.2:2181,zk.host.3:2181 +serializer.class=kafka.serializer.StringEncoder +request.required.acks=1 \ No newline at end of file diff --git a/samples/KafkaConsistentRegionConsumerSimple/info.xml b/samples/KafkaConsistentRegionConsumerSimple/info.xml new file mode 100644 index 0000000..6149e3f --- /dev/null +++ b/samples/KafkaConsistentRegionConsumerSimple/info.xml @@ -0,0 +1,15 @@ + + + + ConsistentRegionConsumerSimple + + 1.0.0 + 4.0.1.0 + + + + com.ibm.streamsx.messaging + [2.0.2,3.0.0) + + + \ No newline at end of file From 8d43bd91e207370e865af6b75076cc26b2c97783 Mon Sep 17 00:00:00 2001 From: Alex Cook Date: Tue, 11 Aug 2015 11:54:52 -0400 Subject: [PATCH 13/14] Remove bad classpath requirements. --- com.ibm.streamsx.messaging/.classpath | 8 ++++---- .../src/com/ibm/streamsx/messaging/kafka/KafkaSource.java | 2 +- samples/KafkaConsistentRegionConsumerParallel/.classpath | 3 +-- samples/KafkaConsistentRegionConsumerSimple/.classpath | 3 +-- samples/KafkaConsistentRegionConsumerSimple/.gitignore | 1 + 5 files changed, 8 insertions(+), 9 deletions(-) create mode 100644 samples/KafkaConsistentRegionConsumerSimple/.gitignore diff --git a/com.ibm.streamsx.messaging/.classpath b/com.ibm.streamsx.messaging/.classpath index 540b7d2..c8253e9 100644 --- a/com.ibm.streamsx.messaging/.classpath +++ b/com.ibm.streamsx.messaging/.classpath @@ -6,10 +6,8 @@ - - @@ -18,8 +16,10 @@ - - + + + + diff --git a/com.ibm.streamsx.messaging/impl/java/src/com/ibm/streamsx/messaging/kafka/KafkaSource.java b/com.ibm.streamsx.messaging/impl/java/src/com/ibm/streamsx/messaging/kafka/KafkaSource.java index 0e7e2d2..1a28c7e 100644 --- a/com.ibm.streamsx.messaging/impl/java/src/com/ibm/streamsx/messaging/kafka/KafkaSource.java +++ b/com.ibm.streamsx.messaging/impl/java/src/com/ibm/streamsx/messaging/kafka/KafkaSource.java @@ -158,7 +158,7 @@ public void setConnectionRetryInterval(int value) { "Note that there may be multiple threads receiving messages depending on the configuration specified. " + "Ordering of messages is not guaranteed." + "\\n\\n**Behavior in a Consistent Region**" + - "\\nThis operator cannot be used inside a consistent region." + "\\nThis operator can be used inside a consistent region. some behavior." ; @Override diff --git a/samples/KafkaConsistentRegionConsumerParallel/.classpath b/samples/KafkaConsistentRegionConsumerParallel/.classpath index cbb0e4c..36e3d95 100644 --- a/samples/KafkaConsistentRegionConsumerParallel/.classpath +++ b/samples/KafkaConsistentRegionConsumerParallel/.classpath @@ -1,6 +1,5 @@ - @@ -8,5 +7,5 @@ - + diff --git a/samples/KafkaConsistentRegionConsumerSimple/.classpath b/samples/KafkaConsistentRegionConsumerSimple/.classpath index cbb0e4c..36e3d95 100644 --- a/samples/KafkaConsistentRegionConsumerSimple/.classpath +++ b/samples/KafkaConsistentRegionConsumerSimple/.classpath @@ -1,6 +1,5 @@ - @@ -8,5 +7,5 @@ - + diff --git a/samples/KafkaConsistentRegionConsumerSimple/.gitignore b/samples/KafkaConsistentRegionConsumerSimple/.gitignore new file mode 100644 index 0000000..01afdcf --- /dev/null +++ b/samples/KafkaConsistentRegionConsumerSimple/.gitignore @@ -0,0 +1 @@ +/impl/ From c3e26972fcd4ba7ca0d4388288497cf1a105e8aa Mon Sep 17 00:00:00 2001 From: Alex Cook Date: Tue, 11 Aug 2015 14:32:11 -0400 Subject: [PATCH 14/14] Updated documentation. --- .../ibm/streamsx/messaging/kafka/KafkaBaseOper.java | 4 ++-- .../ibm/streamsx/messaging/kafka/KafkaSource.java | 12 +++++++----- .../messaging/kafka/SimpleConsumerClient.java | 4 ++-- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/com.ibm.streamsx.messaging/impl/java/src/com/ibm/streamsx/messaging/kafka/KafkaBaseOper.java b/com.ibm.streamsx.messaging/impl/java/src/com/ibm/streamsx/messaging/kafka/KafkaBaseOper.java index 9b790ea..729e291 100644 --- a/com.ibm.streamsx.messaging/impl/java/src/com/ibm/streamsx/messaging/kafka/KafkaBaseOper.java +++ b/com.ibm.streamsx.messaging/impl/java/src/com/ibm/streamsx/messaging/kafka/KafkaBaseOper.java @@ -114,12 +114,12 @@ public String getPropertiesFile() { return propertiesFile; } - @Parameter(optional = true, description = "Name of the attribute for the message. This attribute is required. Default is \\\"message\\\"") + @Parameter(optional = true, description = "Name of the attribute for the message. This attribute is required. Default is \\\"message\\\".") public void setMessageAttribute(String value) { messageAH.setName(value); } - @Parameter(optional = true, description = "Name of the attribute for the key. Default is \\\"key\\\"") + @Parameter(optional = true, description = "Name of the attribute for the key. Default is \\\"key\\\".") public void setKeyAttribute(String value) { keyAH.setName(value); } diff --git a/com.ibm.streamsx.messaging/impl/java/src/com/ibm/streamsx/messaging/kafka/KafkaSource.java b/com.ibm.streamsx.messaging/impl/java/src/com/ibm/streamsx/messaging/kafka/KafkaSource.java index 1a28c7e..f0c2668 100644 --- a/com.ibm.streamsx.messaging/impl/java/src/com/ibm/streamsx/messaging/kafka/KafkaSource.java +++ b/com.ibm.streamsx.messaging/impl/java/src/com/ibm/streamsx/messaging/kafka/KafkaSource.java @@ -130,7 +130,7 @@ public void setTopic(List values) { } @Parameter(name="partition", optional=true, - description="Partition to subscribe to.") + description="Partition to subscribe to. This must be set when using consistent region.") public void setPartition(int value) { this.a_partition = value; } @@ -142,23 +142,25 @@ public void setTriggerCount(int value) { } @Parameter(name="leaderConnectionRetries", optional=true, - description="Number of attempts at finding a Kafka Broker before giving up. This is relevant when first looking for a broker, and in the case that a lead broker host goes down. Default is 3.") + description="Number of attempts at finding a Kafka Broker before giving up. This is relevant when first looking for a broker, and in the case that a lead broker host goes down. This is only valid when the partition parameter is set. Default is 3.") public void setLeaderConnectionRetries(int value) { this.leaderConnectionRetries = value; } @Parameter(name="connectionRetryInterval", optional=true, - description="Interval between each attempt to find a lead Kafka Broker in milliseconds. Number of attempts is set by the leaderConnectionRetries parameter.") + description="Interval between each attempt to find a lead Kafka Broker in milliseconds. Number of attempts is set by the leaderConnectionRetries parameter. This is only valid when the partition parameter is set. Default is 1000 ms.") public void setConnectionRetryInterval(int value) { this.connectionRetryInterval = value; } public static final String DESC = - "This operator acts as a Kafka consumer receiving messages for one or more topics. " + + "This operator acts as a Kafka consumer receiving messages for a single topic. " + "Note that there may be multiple threads receiving messages depending on the configuration specified. " + "Ordering of messages is not guaranteed." + "\\n\\n**Behavior in a Consistent Region**" + - "\\nThis operator can be used inside a consistent region. some behavior." + "\\nThis operator can be used inside a consistent region. Operator driven and periodical checkpointing " + + "are supported. Partition to be read from must be specified. To consume multiple partitions in a topic, use " + + "user defined parallelism or multiple consumers." ; @Override diff --git a/com.ibm.streamsx.messaging/impl/java/src/com/ibm/streamsx/messaging/kafka/SimpleConsumerClient.java b/com.ibm.streamsx.messaging/impl/java/src/com/ibm/streamsx/messaging/kafka/SimpleConsumerClient.java index dd1f2a7..0d282fe 100644 --- a/com.ibm.streamsx.messaging/impl/java/src/com/ibm/streamsx/messaging/kafka/SimpleConsumerClient.java +++ b/com.ibm.streamsx.messaging/impl/java/src/com/ibm/streamsx/messaging/kafka/SimpleConsumerClient.java @@ -43,7 +43,7 @@ public class SimpleConsumerClient implements StateHandler { static final String OPER_NAME = "KafkaConsumer"; private ConsistentRegionContext crContext; private boolean shutdown = false; - private long triggerCount; + private int triggerCount; private long triggerIteration = 0; private OperatorContext operContext; private boolean inReset = false; @@ -72,7 +72,7 @@ public class SimpleConsumerClient implements StateHandler { public SimpleConsumerClient(String a_topic, int a_partition, AttributeHelper keyAH, - AttributeHelper messageAH, Properties props, long trigCnt, + AttributeHelper messageAH, Properties props, int trigCnt, int connectionRetries, int connectionRetryInterval) { this.topic = a_topic; this.keyAH = keyAH;