From 48ec5779bdaddd55154dca69e2708709056f51aa Mon Sep 17 00:00:00 2001 From: nhne Date: Tue, 4 Jul 2017 16:23:01 +0900 Subject: [PATCH 01/34] [REEF-362] Implement a Wake Transport using HTTP Additional implementation for Wake via HTTP JIRA: [REEF-362](https://issues.apache.org/jira/browse/REEF-362) Pull Request This closes # --- .../netty/AbstractNettyEventListener.java | 8 +- .../remote/transport/netty/LinkReference.java | 12 +- .../netty/NettyChannelHandlerFactory.java | 2 +- .../transport/netty/NettyEventListener.java | 2 +- .../http/HttpMessagingTransportFactory.java | 138 ++++++ .../netty/http/NettyHttpChannelHandler.java | 128 ++++++ .../http/NettyHttpChannelHandlerFactory.java | 51 +++ .../http/NettyHttpChannelInitializer.java | 70 +++ .../http/NettyHttpClientEventListener.java | 101 +++++ .../transport/netty/http/NettyHttpLink.java | 154 +++++++ .../http/NettyHttpMessagingTransport.java | 415 ++++++++++++++++++ .../http/NettyHttpServerEventListener.java | 159 +++++++ .../transport/netty/http/package-info.java | 22 + .../wake/test/remote/TransportHttpTest.java | 162 +++++++ 14 files changed, 1412 insertions(+), 12 deletions(-) create mode 100644 lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/http/HttpMessagingTransportFactory.java create mode 100644 lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/http/NettyHttpChannelHandler.java create mode 100644 lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/http/NettyHttpChannelHandlerFactory.java create mode 100644 lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/http/NettyHttpChannelInitializer.java create mode 100644 lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/http/NettyHttpClientEventListener.java create mode 100644 lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/http/NettyHttpLink.java create mode 100644 lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/http/NettyHttpMessagingTransport.java create mode 100644 lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/http/NettyHttpServerEventListener.java create mode 100644 lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/http/package-info.java create mode 100644 lang/java/reef-wake/wake/src/test/java/org/apache/reef/wake/test/remote/TransportHttpTest.java diff --git a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/AbstractNettyEventListener.java b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/AbstractNettyEventListener.java index d286d0c553..823fd3b5d8 100644 --- a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/AbstractNettyEventListener.java +++ b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/AbstractNettyEventListener.java @@ -33,7 +33,7 @@ * Generic functionality for the Netty event listener. * This is a base class for client and server versions. */ -abstract class AbstractNettyEventListener implements NettyEventListener { +public abstract class AbstractNettyEventListener implements NettyEventListener { protected static final Logger LOG = Logger.getLogger(AbstractNettyEventListener.class.getName()); @@ -41,9 +41,9 @@ abstract class AbstractNettyEventListener implements NettyEventListener { protected final EStage stage; protected EventHandler exceptionHandler; - AbstractNettyEventListener( - final ConcurrentMap addrToLinkRefMap, - final EStage stage) { + public AbstractNettyEventListener( + final ConcurrentMap addrToLinkRefMap, + final EStage stage) { this.addrToLinkRefMap = addrToLinkRefMap; this.stage = stage; } diff --git a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/LinkReference.java b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/LinkReference.java index 6e780b888a..6afe136587 100644 --- a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/LinkReference.java +++ b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/LinkReference.java @@ -26,27 +26,27 @@ * A reference for a link. * When channel became active, LinkReference is created and mapped with remote address. */ -final class LinkReference { +public final class LinkReference { private final AtomicInteger connectInProgress = new AtomicInteger(0); private Link link; - LinkReference() { + public LinkReference() { } - LinkReference(final Link link) { + public LinkReference(final Link link) { this.link = link; } - synchronized Link getLink() { + public synchronized Link getLink() { return this.link; } - synchronized void setLink(final Link link) { + public synchronized void setLink(final Link link) { this.link = link; } - AtomicInteger getConnectInProgress() { + public AtomicInteger getConnectInProgress() { return this.connectInProgress; } } diff --git a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyChannelHandlerFactory.java b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyChannelHandlerFactory.java index 912a6074f0..665bb195a0 100644 --- a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyChannelHandlerFactory.java +++ b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyChannelHandlerFactory.java @@ -24,7 +24,7 @@ /** * Factory that creates a Netty channel handler. */ -interface NettyChannelHandlerFactory { +public interface NettyChannelHandlerFactory { /** * Creates a channel inbound handler. diff --git a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyEventListener.java b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyEventListener.java index a6a7359672..faa9e1eebb 100644 --- a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyEventListener.java +++ b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyEventListener.java @@ -23,7 +23,7 @@ /** * Netty event listener. */ -interface NettyEventListener { +public interface NettyEventListener { /** * Handles the message. diff --git a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/http/HttpMessagingTransportFactory.java b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/http/HttpMessagingTransportFactory.java new file mode 100644 index 0000000000..cf247221d9 --- /dev/null +++ b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/http/HttpMessagingTransportFactory.java @@ -0,0 +1,138 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.reef.wake.remote.transport.netty.http; + +import org.apache.reef.tang.Injector; +import org.apache.reef.tang.Tang; +import org.apache.reef.tang.exceptions.InjectionException; +import org.apache.reef.wake.EStage; +import org.apache.reef.wake.EventHandler; +import org.apache.reef.wake.impl.SyncStage; +import org.apache.reef.wake.remote.RemoteConfiguration; +import org.apache.reef.wake.remote.address.LocalAddressProvider; +import org.apache.reef.wake.remote.impl.TransportEvent; +import org.apache.reef.wake.remote.ports.TcpPortProvider; +import org.apache.reef.wake.remote.transport.Transport; +import org.apache.reef.wake.remote.transport.TransportFactory; + +import javax.inject.Inject; + +/** + * Factory that creates a messaging transport. + */ +public final class HttpMessagingTransportFactory implements TransportFactory { + + private final String localAddress; + + @Inject + private HttpMessagingTransportFactory(final LocalAddressProvider localAddressProvider) { + this.localAddress = localAddressProvider.getLocalAddress(); + } + + /** + * Creates a transport. + * + * @param port a listening port + * @param clientHandler a transport client side handler + * @param serverHandler a transport server side handler + * @param exHandler a exception handler + */ + @Override + public Transport newInstance(final int port, + final EventHandler clientHandler, + final EventHandler serverHandler, + final EventHandler exHandler) { + + final Injector injector = Tang.Factory.getTang().newInjector(); + injector.bindVolatileParameter(RemoteConfiguration.HostAddress.class, this.localAddress); + injector.bindVolatileParameter(RemoteConfiguration.Port.class, port); + injector.bindVolatileParameter(RemoteConfiguration.RemoteClientStage.class, new SyncStage<>(clientHandler)); + injector.bindVolatileParameter(RemoteConfiguration.RemoteServerStage.class, new SyncStage<>(serverHandler)); + + final Transport transport; + try { + transport = injector.getInstance(NettyHttpMessagingTransport.class); + transport.registerErrorHandler(exHandler); + return transport; + } catch (final InjectionException e) { + throw new RuntimeException(e); + } + } + + /** + * Creates a transport. + * + * @param hostAddress a host address + * @param port a listening port + * @param clientStage a client stage + * @param serverStage a server stage + * @param numberOfTries a number of tries + * @param retryTimeout a timeout for retry + */ + @Override + public Transport newInstance(final String hostAddress, + final int port, + final EStage clientStage, + final EStage serverStage, + final int numberOfTries, + final int retryTimeout) { + try { + TcpPortProvider tcpPortProvider = Tang.Factory.getTang().newInjector().getInstance(TcpPortProvider.class); + return newInstance(hostAddress, port, clientStage, + serverStage, numberOfTries, retryTimeout, tcpPortProvider); + } catch (final InjectionException e) { + throw new RuntimeException(e); + } + } + + /** + * Creates a transport. + * + * @param hostAddress a host address + * @param port a listening port + * @param clientStage a client stage + * @param serverStage a server stage + * @param numberOfTries a number of tries + * @param retryTimeout a timeout for retry + * @param tcpPortProvider a provider for TCP port + */ + @Override + public Transport newInstance(final String hostAddress, + final int port, + final EStage clientStage, + final EStage serverStage, + final int numberOfTries, + final int retryTimeout, + final TcpPortProvider tcpPortProvider) { + + final Injector injector = Tang.Factory.getTang().newInjector(); + injector.bindVolatileParameter(RemoteConfiguration.HostAddress.class, hostAddress); + injector.bindVolatileParameter(RemoteConfiguration.Port.class, port); + injector.bindVolatileParameter(RemoteConfiguration.RemoteClientStage.class, clientStage); + injector.bindVolatileParameter(RemoteConfiguration.RemoteServerStage.class, serverStage); + injector.bindVolatileParameter(RemoteConfiguration.NumberOfTries.class, numberOfTries); + injector.bindVolatileParameter(RemoteConfiguration.RetryTimeout.class, retryTimeout); + injector.bindVolatileInstance(TcpPortProvider.class, tcpPortProvider); + try { + return injector.getInstance(NettyHttpMessagingTransport.class); + } catch (final InjectionException e) { + throw new RuntimeException(e); + } + } +} diff --git a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/http/NettyHttpChannelHandler.java b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/http/NettyHttpChannelHandler.java new file mode 100644 index 0000000000..7af5a60745 --- /dev/null +++ b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/http/NettyHttpChannelHandler.java @@ -0,0 +1,128 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.reef.wake.remote.transport.netty.http; + +import io.netty.channel.Channel; +import io.netty.channel.ChannelHandlerContext; +import io.netty.channel.ChannelInboundHandlerAdapter; +import io.netty.channel.group.ChannelGroup; +import io.netty.util.ReferenceCountUtil; +import org.apache.reef.wake.remote.transport.netty.NettyEventListener; + +import java.util.logging.Level; +import java.util.logging.Logger; + +/** + * Netty channel handler for channel status(active/inactive) and incoming data. + */ +final class NettyHttpChannelHandler extends ChannelInboundHandlerAdapter { + + private static final Logger LOG = Logger.getLogger(NettyHttpChannelHandler.class.getName()); + + private final String tag; + private final ChannelGroup channelGroup; + private final NettyEventListener listener; + + /** + * Constructs a Netty channel handler. + * + * @param tag tag string + * @param channelGroup the channel group + * @param listener the Netty event listener + */ + NettyHttpChannelHandler( + final String tag, final ChannelGroup channelGroup, final NettyEventListener listener) { + this.tag = tag; + this.channelGroup = channelGroup; + this.listener = listener; + } + + /** + * Handle the incoming message: pass it to the listener. + * + * @param ctx the context object for this handler. + * @param msg the message. + * @throws Exception + */ + @Override + public void channelRead( + final ChannelHandlerContext ctx, final Object msg) throws Exception { + try { + this.listener.channelRead(ctx, msg); + } finally { + ReferenceCountUtil.release(msg); + } + } + + /** + * Flushes after channel read complete. + * + * @param ctx + * @throws Exception + */ + + @Override + public void channelReadComplete(final ChannelHandlerContext ctx) throws Exception { + ctx.flush(); + } + + /** + * Handles the channel active event. + * + * @param ctx the context object for this handler + * @throws Exception + */ + @Override + public void channelActive(final ChannelHandlerContext ctx) throws Exception { + this.channelGroup.add(ctx.channel()); + this.listener.channelActive(ctx); + super.channelActive(ctx); + } + + /** + * Handles the channel inactive event. + * + * @param ctx the context object for this handler + * @throws Exception + */ + @Override + public void channelInactive(final ChannelHandlerContext ctx) throws Exception { + this.listener.channelInactive(ctx); + super.channelInactive(ctx); + } + + /** + * Handles the exception event. + * + * @param ctx the context object for this handler + * @param cause the cause + * @throws Exception + */ + @Override + public void exceptionCaught(final ChannelHandlerContext ctx, final Throwable cause) { + final Channel channel = ctx.channel(); + + LOG.log(Level.INFO, "Unexpected exception from downstream. channel: {0} local: {1} remote: {2}", + new Object[]{channel, channel.localAddress(), channel.remoteAddress()}); + LOG.log(Level.WARNING, "Unexpected exception from downstream.", cause); + channel.close(); + this.listener.exceptionCaught(ctx, cause); + } + +} diff --git a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/http/NettyHttpChannelHandlerFactory.java b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/http/NettyHttpChannelHandlerFactory.java new file mode 100644 index 0000000000..5df6cee35e --- /dev/null +++ b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/http/NettyHttpChannelHandlerFactory.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.reef.wake.remote.transport.netty.http; + +import io.netty.channel.ChannelInboundHandlerAdapter; +import io.netty.channel.group.ChannelGroup; +import org.apache.reef.wake.remote.transport.netty.NettyChannelHandlerFactory; +import org.apache.reef.wake.remote.transport.netty.NettyEventListener; + +/** + * Default Netty channel handler factory. + */ +final class NettyHttpChannelHandlerFactory implements NettyChannelHandlerFactory { + + private final String tag; + private final ChannelGroup group; + private final NettyEventListener listener; + + NettyHttpChannelHandlerFactory( + final String tag, final ChannelGroup group, final NettyEventListener listener) { + this.tag = tag; + this.group = group; + this.listener = listener; + } + + /** + * Creates a Netty channel handler. + * + * @return a simple channel upstream handler. + */ + @Override + public ChannelInboundHandlerAdapter createChannelInboundHandler() { + return new NettyHttpChannelHandler(this.tag, this.group, this.listener); + } +} diff --git a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/http/NettyHttpChannelInitializer.java b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/http/NettyHttpChannelInitializer.java new file mode 100644 index 0000000000..c578ae6432 --- /dev/null +++ b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/http/NettyHttpChannelInitializer.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.reef.wake.remote.transport.netty.http; + +import io.netty.channel.ChannelInitializer; +import io.netty.channel.socket.SocketChannel; +import io.netty.handler.codec.http.*; +import io.netty.handler.ssl.SslContext; +import org.apache.reef.wake.remote.transport.netty.NettyChannelHandlerFactory; + +/** + * Netty Http channel initializer for Transport. + */ +class NettyHttpChannelInitializer extends ChannelInitializer { + + static final int TYPE_SERVER = 0; + static final int TYPE_CLIENT = 1; + + private final NettyChannelHandlerFactory handlerFactory; + private final SslContext sslContext; + private final int type; + + NettyHttpChannelInitializer( + final NettyChannelHandlerFactory handlerFactory, + final SslContext sslContext, + final int type) { + if(type != TYPE_SERVER && type != TYPE_CLIENT){ + throw new IllegalArgumentException("Invalid type of channel"); + } + this.handlerFactory = handlerFactory; + this.sslContext = sslContext; + this.type = type; + } + + @Override + protected void initChannel(final SocketChannel ch) throws Exception { + if(sslContext != null) { + ch.pipeline().addLast(sslContext.newHandler(ch.alloc())); + } + if(type == TYPE_SERVER) { + // Init channel for server + ch.pipeline() + .addLast("codec", new HttpServerCodec()) + .addLast("requestDecoder", new HttpRequestDecoder()) + .addLast("responseEncoder", new HttpResponseEncoder()); + } else if (type == TYPE_CLIENT) { + // Init channel for client + ch.pipeline() + .addLast("codec", new HttpClientCodec()) + .addLast("decompressor", new HttpContentDecompressor()); + } + ch.pipeline().addLast("handler", handlerFactory.createChannelInboundHandler()); + } +} diff --git a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/http/NettyHttpClientEventListener.java b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/http/NettyHttpClientEventListener.java new file mode 100644 index 0000000000..a8c822b951 --- /dev/null +++ b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/http/NettyHttpClientEventListener.java @@ -0,0 +1,101 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.reef.wake.remote.transport.netty.http; + +import io.netty.buffer.ByteBuf; +import io.netty.channel.Channel; +import io.netty.channel.ChannelHandlerContext; +import io.netty.handler.codec.http.HttpContent; +import io.netty.handler.codec.http.HttpResponse; +import io.netty.handler.codec.http.LastHttpContent; +import io.netty.util.CharsetUtil; +import org.apache.reef.wake.EStage; +import org.apache.reef.wake.remote.impl.TransportEvent; +import org.apache.reef.wake.remote.transport.netty.AbstractNettyEventListener; +import org.apache.reef.wake.remote.transport.netty.LinkReference; + +import java.net.SocketAddress; +import java.util.concurrent.ConcurrentMap; +import java.util.logging.Level; +import java.util.logging.Logger; + +/** + * A Netty event listener for server. + */ +final class NettyHttpClientEventListener extends AbstractNettyEventListener { + + private static final Logger LOG = Logger.getLogger(NettyHttpClientEventListener.class.getName()); + private StringBuilder buf = new StringBuilder(); + + NettyHttpClientEventListener( + final ConcurrentMap addrToLinkRefMap, + final EStage stage) { + super(addrToLinkRefMap, stage); + } + + @Override + public void channelRead(final ChannelHandlerContext ctx, final Object msg) { + if (msg instanceof HttpResponse) { + System.out.println("CLIENT : IT IS HTTP RESPONSE"); + } else if (msg instanceof HttpContent) { + System.out.println("CLIENT : It is HttpContent"); + HttpContent httpContent = (HttpContent) msg; + ByteBuf content = httpContent.content(); + if (content.isReadable()) { + buf.append("CONTENT: "); + buf.append(content.toString(CharsetUtil.UTF_8)); + buf.append("\r\n"); + } + + if (msg instanceof LastHttpContent) { + final Channel channel = ctx.channel(); + byte[] message = new byte[content.readableBytes()]; + content.readBytes(message); + if (LOG.isLoggable(Level.FINEST)) { + LOG.log(Level.FINEST, "MessageEvent: local: {0} remote: {1} :: {2}", new Object[]{ + channel.localAddress(), channel.remoteAddress(), buf}); + } + + if (message.length > 0) { + // send to the dispatch stage + this.stage.onNext(this.getTransportEvent(message, channel)); + } + } + } else { + LOG.log(Level.SEVERE, "Unknown type of message received: {0}", msg); + } + + } + + @Override + public void channelActive(final ChannelHandlerContext ctx) { + // noop + LOG.log(Level.FINEST, "{0}", ctx); + } + + @Override + protected TransportEvent getTransportEvent(final byte[] message, final Channel channel) { + return new TransportEvent(message, channel.localAddress(), channel.remoteAddress()); + } + + @Override + protected void exceptionCleanup(final ChannelHandlerContext ctx, final Throwable cause) { + this.closeChannel(ctx.channel()); + } +} diff --git a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/http/NettyHttpLink.java b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/http/NettyHttpLink.java new file mode 100644 index 0000000000..25ce9ddbb4 --- /dev/null +++ b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/http/NettyHttpLink.java @@ -0,0 +1,154 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.reef.wake.remote.transport.netty.http; + +import io.netty.buffer.ByteBuf; +import io.netty.buffer.Unpooled; +import io.netty.channel.Channel; +import io.netty.channel.ChannelFuture; +import io.netty.channel.ChannelFutureListener; +import io.netty.handler.codec.http.*; +import org.apache.reef.wake.remote.Encoder; +import org.apache.reef.wake.remote.transport.Link; +import org.apache.reef.wake.remote.transport.LinkListener; + +import java.net.SocketAddress; +import java.net.URI; +import java.util.logging.Level; +import java.util.logging.Logger; + +/** + * Link implementation with Netty Http. + * + * If you set a {@code LinkListener}, it keeps message until writeAndFlush operation completes + * and notifies whether the sent message transferred successfully through the listener. + */ +public class NettyHttpLink implements Link { + + public static final int INT_SIZE = Integer.SIZE / Byte.SIZE; + + private static final Logger LOG = Logger.getLogger(NettyHttpLink.class.getName()); + + private final Channel channel; + private final Encoder encoder; + private final LinkListener listener; + private final URI uri; + + /** + * Constructs a link. + * + * @param channel the channel + * @param encoder the encoder + */ + public NettyHttpLink(final Channel channel, final Encoder encoder) { + this(channel, encoder, null, URI.create("http://127.0.0.1")); + } + + /** + * Constructs a link. + * + * @param channel the channel + * @param encoder the encoder + * @param listener the link listener + * @param uri the URI + */ + public NettyHttpLink( + final Channel channel, + final Encoder encoder, + final LinkListener listener, + final URI uri) { + this.channel = channel; + this.encoder = encoder; + this.listener = listener; + this.uri = uri; + } + + /** + * Writes the message to this link via HTTP. + * + * @param message the message + */ + @Override + public void write(final T message) { + + try { + LOG.log(Level.FINE, "write {0} :: {1}", new Object[] {channel, message}); + FullHttpRequest request = new DefaultFullHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.POST, uri.getRawPath()); + request.headers().set(HttpHeaders.Names.HOST, uri.getHost()); + request.headers().set(HttpHeaders.Names.CONNECTION, HttpHeaders.Values.KEEP_ALIVE); + request.headers().set(HttpHeaders.Names.ACCEPT_ENCODING, HttpHeaders.Values.GZIP); + request.headers().set(HttpHeaders.Names.CONTENT_TYPE, "application/wake-transport"); + ByteBuf buf = Unpooled.copiedBuffer(encoder.encode(message)); + request.headers().set(HttpHeaders.Names.CONTENT_LENGTH, buf.readableBytes()); + request.content().clear().writeBytes(buf); + final ChannelFuture future = channel.writeAndFlush(request); + future.sync(); + if (listener != null) { + future.addListener(new NettyHttpChannelFutureListener<>(message, listener)); + } + } catch (InterruptedException ex) { + LOG.log(Level.SEVERE, "Cannot send request to " + uri.getHost(), ex); + } + } + + /** + * Gets a local address of the link. + * + * @return a local socket address + */ + @Override + public SocketAddress getLocalAddress() { + return channel.localAddress(); + } + + /** + * Gets a remote address of the link. + * + * @return a remote socket address + */ + @Override + public SocketAddress getRemoteAddress() { + return channel.remoteAddress(); + } + + @Override + public String toString() { + return "NettyHttpLink: " + channel; // Channel has good .toString() implementation + } +} + +class NettyHttpChannelFutureListener implements ChannelFutureListener { + + private final T message; + private LinkListener listener; + + NettyHttpChannelFutureListener(final T message, final LinkListener listener) { + this.message = message; + this.listener = listener; + } + + @Override + public void operationComplete(final ChannelFuture channelFuture) throws Exception { + if (channelFuture.isSuccess()) { + listener.onSuccess(message); + } else { + listener.onException(channelFuture.cause(), channelFuture.channel().remoteAddress(), message); + } + } +} diff --git a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/http/NettyHttpMessagingTransport.java b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/http/NettyHttpMessagingTransport.java new file mode 100644 index 0000000000..da116c27d6 --- /dev/null +++ b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/http/NettyHttpMessagingTransport.java @@ -0,0 +1,415 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.reef.wake.remote.transport.netty.http; + +import io.netty.bootstrap.Bootstrap; +import io.netty.bootstrap.ServerBootstrap; +import io.netty.channel.Channel; +import io.netty.channel.ChannelFuture; +import io.netty.channel.ChannelOption; +import io.netty.channel.EventLoopGroup; +import io.netty.channel.group.ChannelGroup; +import io.netty.channel.group.ChannelGroupFuture; +import io.netty.channel.group.DefaultChannelGroup; +import io.netty.channel.nio.NioEventLoopGroup; +import io.netty.channel.socket.nio.NioServerSocketChannel; +import io.netty.channel.socket.nio.NioSocketChannel; +import io.netty.handler.ssl.SslContext; +import io.netty.handler.ssl.util.SelfSignedCertificate; +import io.netty.util.concurrent.Future; +import io.netty.util.concurrent.GlobalEventExecutor; +import org.apache.reef.tang.annotations.Parameter; +import org.apache.reef.wake.EStage; +import org.apache.reef.wake.EventHandler; +import org.apache.reef.wake.impl.DefaultThreadFactory; +import org.apache.reef.wake.remote.Encoder; +import org.apache.reef.wake.remote.RemoteConfiguration; +import org.apache.reef.wake.remote.address.LocalAddressProvider; +import org.apache.reef.wake.remote.exception.RemoteRuntimeException; +import org.apache.reef.wake.remote.impl.TransportEvent; +import org.apache.reef.wake.remote.ports.TcpPortProvider; +import org.apache.reef.wake.remote.transport.Link; +import org.apache.reef.wake.remote.transport.LinkListener; +import org.apache.reef.wake.remote.transport.Transport; +import org.apache.reef.wake.remote.transport.exception.TransportRuntimeException; +import org.apache.reef.wake.remote.transport.netty.LinkReference; + +import javax.inject.Inject; +import java.io.IOException; +import java.net.*; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.logging.Level; +import java.util.logging.Logger; + +import static org.apache.reef.wake.remote.transport.netty.http.NettyHttpChannelInitializer.TYPE_CLIENT; +import static org.apache.reef.wake.remote.transport.netty.http.NettyHttpChannelInitializer.TYPE_SERVER; + +/** + * Messaging transport implementation with Netty HTTP. + */ +public final class NettyHttpMessagingTransport implements Transport { + + /** + * Indicates a hostname that isn't set or known. + */ + public static final String UNKNOWN_HOST_NAME = "##UNKNOWN##"; + + private static final String CLASS_NAME = NettyHttpMessagingTransport.class.getSimpleName(); + + private static final Logger LOG = Logger.getLogger(CLASS_NAME); + + private static final int SERVER_BOSS_NUM_THREADS = 3; + private static final int SERVER_WORKER_NUM_THREADS = 20; + private static final int CLIENT_WORKER_NUM_THREADS = 10; + + private final ConcurrentMap addrToLinkRefMap = new ConcurrentHashMap<>(); + + private final EventLoopGroup clientWorkerGroup; + private final EventLoopGroup serverBossGroup; + private final EventLoopGroup serverWorkerGroup; + + private final Bootstrap clientBootstrap; + private final ServerBootstrap serverBootstrap; + private final Channel acceptor; + + private final ChannelGroup clientChannelGroup = new DefaultChannelGroup(GlobalEventExecutor.INSTANCE); + private final ChannelGroup serverChannelGroup = new DefaultChannelGroup(GlobalEventExecutor.INSTANCE); + + private final int serverPort; + private final SocketAddress localAddress; + + private final boolean ssl = System.getProperty("ssl") != null; + private final URI uri; + + private final NettyHttpClientEventListener clientEventListener; + private final NettyHttpServerEventListener serverEventListener; + + private final int numberOfTries; + private final int retryTimeout; + + /** + * Constructs a messaging transport. + * + * @param hostAddress the server host address + * @param port the server listening port; when it is 0, randomly assign a port number + * @param clientStage the client-side stage that handles transport events + * @param serverStage the server-side stage that handles transport events + * @param numberOfTries the number of tries of connection + * @param retryTimeout the timeout of reconnection + * @param tcpPortProvider gives an iterator that produces random tcp ports in a range + */ + @Inject + private NettyHttpMessagingTransport( + @Parameter(RemoteConfiguration.HostAddress.class) final String hostAddress, + @Parameter(RemoteConfiguration.Port.class) final int port, + @Parameter(RemoteConfiguration.RemoteClientStage.class) final EStage clientStage, + @Parameter(RemoteConfiguration.RemoteServerStage.class) final EStage serverStage, + @Parameter(RemoteConfiguration.NumberOfTries.class) final int numberOfTries, + @Parameter(RemoteConfiguration.RetryTimeout.class) final int retryTimeout, + final TcpPortProvider tcpPortProvider, + final LocalAddressProvider localAddressProvider) { + + int p = port; + if (p < 0) { + throw new RemoteRuntimeException("Invalid server port: " + p); + } + + final String host = UNKNOWN_HOST_NAME.equals(hostAddress) ? localAddressProvider.getLocalAddress() : hostAddress; + + final SslContext sslContextClient; + final SslContext sslContextServer; + if(ssl){ + try { + SelfSignedCertificate ssc = new SelfSignedCertificate(); + sslContextClient = SslContext.newServerContext(ssc.certificate(), ssc.privateKey()); + sslContextServer = SslContext.newClientContext(); + LOG.log(Level.FINE, "SSL context created"); + } catch (Exception ex) { + final RuntimeException transportException = + new TransportRuntimeException("Could create SSL Context", ex); + LOG.log(Level.SEVERE, "Cannot create SSL Context", ex); + throw transportException; + } + } else { + LOG.log(Level.FINE, "System property 'ssl' is not set"); + sslContextClient = null; + sslContextServer = null; + } + + this.uri = URI.create(ssl ? "https://" : "http://" + hostAddress); + + this.numberOfTries = numberOfTries; + this.retryTimeout = retryTimeout; + this.clientEventListener = new NettyHttpClientEventListener(this.addrToLinkRefMap, clientStage); + this.serverEventListener = new NettyHttpServerEventListener(this.addrToLinkRefMap, serverStage, this.uri); + + this.serverBossGroup = new NioEventLoopGroup(SERVER_BOSS_NUM_THREADS, + new DefaultThreadFactory(CLASS_NAME + ":ServerBoss")); + this.serverWorkerGroup = new NioEventLoopGroup(SERVER_WORKER_NUM_THREADS, + new DefaultThreadFactory(CLASS_NAME + ":ServerWorker")); + this.clientWorkerGroup = new NioEventLoopGroup(CLIENT_WORKER_NUM_THREADS, + new DefaultThreadFactory(CLASS_NAME + ":ClientWorker")); + + this.clientBootstrap = new Bootstrap(); + this.clientBootstrap.group(this.clientWorkerGroup) + .channel(NioSocketChannel.class) + .handler(new NettyHttpChannelInitializer(new NettyHttpChannelHandlerFactory("client", + this.clientChannelGroup, this.clientEventListener), sslContextClient, TYPE_CLIENT)) + .option(ChannelOption.SO_REUSEADDR, true) + .option(ChannelOption.SO_KEEPALIVE, true); + + this.serverBootstrap = new ServerBootstrap(); + this.serverBootstrap.group(this.serverBossGroup, this.serverWorkerGroup) + .channel(NioServerSocketChannel.class) + .childHandler(new NettyHttpChannelInitializer(new NettyHttpChannelHandlerFactory("server", + this.serverChannelGroup, this.serverEventListener), sslContextServer, TYPE_SERVER)) + .option(ChannelOption.SO_BACKLOG, 128) + .option(ChannelOption.SO_REUSEADDR, true) + .childOption(ChannelOption.SO_KEEPALIVE, true); + + LOG.log(Level.FINE, "Binding to {0}", p); + + Channel acceptorFound = null; + try { + if (p > 0) { + acceptorFound = this.serverBootstrap.bind(new InetSocketAddress(host, p)).sync().channel(); + } else { + final Iterator ports = tcpPortProvider.iterator(); + while (acceptorFound == null) { + if (!ports.hasNext()) { + throw new IllegalStateException("tcpPortProvider cannot find a free port."); + } + p = ports.next(); + LOG.log(Level.FINEST, "Try port {0}", p); + try { + acceptorFound = this.serverBootstrap.bind(new InetSocketAddress(host, p)).sync().channel(); + } catch (final Exception ex) { + if (ex instanceof BindException) { + LOG.log(Level.FINEST, "The port {0} is already bound. Try again", p); + } else { + throw ex; + } + } + } + } + } catch (final IllegalStateException ex) { + final RuntimeException transportException = + new TransportRuntimeException("tcpPortProvider failed to return free ports.", ex); + LOG.log(Level.SEVERE, "Cannot find a free port with " + tcpPortProvider, transportException); + + this.clientWorkerGroup.shutdownGracefully(); + this.serverBossGroup.shutdownGracefully(); + this.serverWorkerGroup.shutdownGracefully(); + throw transportException; + + } catch (final Exception ex) { + final RuntimeException transportException = + new TransportRuntimeException("Cannot bind to port " + p, ex); + LOG.log(Level.SEVERE, "Cannot bind to port " + p, ex); + + this.clientWorkerGroup.shutdownGracefully(); + this.serverBossGroup.shutdownGracefully(); + this.serverWorkerGroup.shutdownGracefully(); + throw transportException; + } + this.acceptor = acceptorFound; + this.serverPort = p; + this.localAddress = new InetSocketAddress(host, this.serverPort); + + LOG.log(Level.FINE, "Starting netty transport socket address: {0}", this.localAddress); + } + + /** + * Closes all channels and releases all resources. + */ + @Override + public void close() { + + LOG.log(Level.FINE, "Closing netty transport socket address: {0}", this.localAddress); + + final ChannelGroupFuture clientChannelGroupFuture = this.clientChannelGroup.close(); + final ChannelGroupFuture serverChannelGroupFuture = this.serverChannelGroup.close(); + final ChannelFuture acceptorFuture = this.acceptor.close(); + + final ArrayList eventLoopGroupFutures = new ArrayList<>(3); + eventLoopGroupFutures.add(this.clientWorkerGroup.shutdownGracefully()); + eventLoopGroupFutures.add(this.serverBossGroup.shutdownGracefully()); + eventLoopGroupFutures.add(this.serverWorkerGroup.shutdownGracefully()); + + clientChannelGroupFuture.awaitUninterruptibly(); + serverChannelGroupFuture.awaitUninterruptibly(); + + try { + acceptorFuture.sync(); + } catch (final Exception ex) { + LOG.log(Level.SEVERE, "Error closing the acceptor channel for " + this.localAddress, ex); + } + + for (final Future eventLoopGroupFuture : eventLoopGroupFutures) { + eventLoopGroupFuture.awaitUninterruptibly(); + } + + LOG.log(Level.FINE, "Closing netty transport socket address: {0} done", this.localAddress); + } + + /** + * Returns a link for the remote address if cached; otherwise opens, caches and returns. + * When it opens a link for the remote address, only one attempt for the address is made at a given time + * + * @param remoteAddr the remote socket address + * @param encoder the encoder + * @param listener the link listener + * @return a link associated with the address + */ + @Override + public Link open(final SocketAddress remoteAddr, final Encoder encoder, + final LinkListener listener) throws IOException { + + Link link = null; + + for (int i = 0; i <= this.numberOfTries; ++i) { + LinkReference linkRef = this.addrToLinkRefMap.get(remoteAddr); + + if (linkRef != null) { + link = (Link) linkRef.getLink(); + if (LOG.isLoggable(Level.FINE)) { + LOG.log(Level.FINE, "Link {0} for {1} found", new Object[]{link, remoteAddr}); + } + if (link != null) { + return link; + } + } + + if (i == this.numberOfTries) { + // Connection failure + throw new ConnectException("Connection to " + remoteAddr + " refused"); + } + + LOG.log(Level.FINE, "No cached link for {0} thread {1}", + new Object[]{remoteAddr, Thread.currentThread()}); + + // no linkRef + final LinkReference newLinkRef = new LinkReference(); + final LinkReference prior = this.addrToLinkRefMap.putIfAbsent(remoteAddr, newLinkRef); + final AtomicInteger flag = prior != null ? + prior.getConnectInProgress() : newLinkRef.getConnectInProgress(); + + synchronized (flag) { + if (!flag.compareAndSet(0, 1)) { + while (flag.get() == 1) { + try { + flag.wait(); + } catch (final InterruptedException ex) { + LOG.log(Level.WARNING, "Wait interrupted", ex); + } + } + } + } + + linkRef = this.addrToLinkRefMap.get(remoteAddr); + link = (Link) linkRef.getLink(); + + if (link != null) { + return link; + } + + ChannelFuture connectFuture = null; + try { + connectFuture = this.clientBootstrap.connect(remoteAddr); + connectFuture.syncUninterruptibly(); + + link = new NettyHttpLink<>(connectFuture.channel(), encoder, listener, this.uri); + linkRef.setLink(link); + + synchronized (flag) { + flag.compareAndSet(1, 2); + flag.notifyAll(); + } + break; + } catch (final Exception e) { + if (e.getClass().getSimpleName().compareTo("ConnectException") == 0) { + LOG.log(Level.WARNING, "Connection refused. Retry {0} of {1}", + new Object[]{i + 1, this.numberOfTries}); + synchronized (flag) { + flag.compareAndSet(1, 0); + flag.notifyAll(); + } + + if (i < this.numberOfTries) { + try { + Thread.sleep(retryTimeout); + } catch (final InterruptedException interrupt) { + LOG.log(Level.WARNING, "Thread {0} interrupted while sleeping", Thread.currentThread()); + } + } + } else { + throw e; + } + } + } + + return link; + } + + /** + * Returns a link for the remote address if already cached; otherwise, returns null. + * + * @param remoteAddr the remote address + * @return a link if already cached; otherwise, null + */ + public Link get(final SocketAddress remoteAddr) { + final LinkReference linkRef = this.addrToLinkRefMap.get(remoteAddr); + return linkRef != null ? (Link) linkRef.getLink() : null; + } + + /** + * Gets a server local socket address of this transport. + * + * @return a server local socket address + */ + @Override + public SocketAddress getLocalAddress() { + return this.localAddress; + } + + /** + * Gets a server listening port of this transport. + * + * @return a listening port number + */ + @Override + public int getListeningPort() { + return this.serverPort; + } + + /** + * Registers the exception event handler. + * + * @param handler the exception event handler + */ + @Override + public void registerErrorHandler(final EventHandler handler) { + this.clientEventListener.registerErrorHandler(handler); + this.serverEventListener.registerErrorHandler(handler); + } +} diff --git a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/http/NettyHttpServerEventListener.java b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/http/NettyHttpServerEventListener.java new file mode 100644 index 0000000000..54bfb19cd4 --- /dev/null +++ b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/http/NettyHttpServerEventListener.java @@ -0,0 +1,159 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.reef.wake.remote.transport.netty.http; + +import io.netty.buffer.ByteBuf; +import io.netty.channel.Channel; +import io.netty.channel.ChannelHandlerContext; +import io.netty.handler.codec.DecoderResult; +import io.netty.handler.codec.http.*; +import io.netty.util.CharsetUtil; +import org.apache.reef.wake.EStage; +import org.apache.reef.wake.remote.impl.ByteCodec; +import org.apache.reef.wake.remote.impl.TransportEvent; +import org.apache.reef.wake.remote.transport.netty.AbstractNettyEventListener; +import org.apache.reef.wake.remote.transport.netty.ByteEncoder; +import org.apache.reef.wake.remote.transport.netty.LinkReference; +import org.apache.reef.wake.remote.transport.netty.LoggingLinkListener; + +import java.net.SocketAddress; +import java.net.URI; +import java.util.Map; +import java.util.concurrent.ConcurrentMap; +import java.util.logging.Level; + +/** + * A Netty event listener for server side. + */ +final class NettyHttpServerEventListener extends AbstractNettyEventListener { + + private HttpRequest httpRequest; + private final StringBuilder buf = new StringBuilder(); + private final URI uri; + + NettyHttpServerEventListener( + final ConcurrentMap addrToLinkRefMap, + final EStage stage, + final URI uri) { + super(addrToLinkRefMap, stage); + this.uri = uri; + } + + + @Override + public void channelActive(final ChannelHandlerContext ctx) { + final Channel channel = ctx.channel(); + + if (LOG.isLoggable(Level.FINEST)) { + LOG.log(Level.FINEST, "Channel active. key: {0}", channel.remoteAddress()); + } + + this.addrToLinkRefMap.putIfAbsent( + channel.remoteAddress(), new LinkReference(new NettyHttpLink<>( + channel, new ByteCodec(), new LoggingLinkListener(), uri))); + + LOG.log(Level.FINER, "Add connected channel ref: {0}", this.addrToLinkRefMap.get(channel.remoteAddress())); + + } + + @Override + public void channelRead(final ChannelHandlerContext ctx, final Object msg) { + final HttpHeaders headers; + final HttpRequest request; + if(msg instanceof HttpRequest) { + LOG.log(Level.FINEST, "HttpRequest received"); + request = (HttpRequest) msg; + this.httpRequest = request; + headers = request.headers(); + if (!headers.isEmpty()) { + for (Map.Entry h: headers) { + CharSequence key = h.getKey(); + CharSequence value = h.getValue(); + buf.append("HEADER: ").append(key).append(" = ").append(value).append("\r\n"); + } + buf.append("\r\n"); + } + appendDecoderResult(buf, request); + } else if (msg instanceof HttpContent) { + LOG.log(Level.FINEST, "HttpContent received"); + HttpContent httpContent = (HttpContent) msg; + ByteBuf content = httpContent.content(); + if (content.isReadable()) { + buf.append("CONTENT: "); + buf.append(content.toString(CharsetUtil.UTF_8)); + buf.append("\r\n"); + appendDecoderResult(buf, this.httpRequest); + } + + if (msg instanceof LastHttpContent) { + buf.append("END OF CONTENT\r\n"); + LastHttpContent trailer = (LastHttpContent) msg; + if (!trailer.trailingHeaders().isEmpty()) { + buf.append("\r\n"); + for (CharSequence name: trailer.trailingHeaders().names()) { + for (CharSequence value: trailer.trailingHeaders().getAll(name)) { + buf.append("TRAILING HEADER: "); + buf.append(name).append(" = ").append(value).append("\r\n"); + } + } + buf.append("\r\n"); + } + + final Channel channel = ctx.channel(); + byte[] message = new byte[content.readableBytes()]; + content.readBytes(message); + if (LOG.isLoggable(Level.FINEST)) { + LOG.log(Level.FINEST, "MessageEvent: local: {0} remote: {1} :: {2}", new Object[]{ + channel.localAddress(), channel.remoteAddress(), content}); + } + + if (message.length > 0) { + // send to the dispatch stage + this.stage.onNext(this.getTransportEvent(message, channel)); + } + } + } else { + LOG.log(Level.SEVERE, "Unknown type of message received: {0}", msg); + } + LOG.log(Level.FINEST, buf.toString()); + } + + private static void appendDecoderResult(final StringBuilder buf, final HttpObject o) { + DecoderResult result = o.getDecoderResult(); + if (result.isSuccess()) { + return; + } + + buf.append(".. WITH DECODER FAILURE: "); + buf.append(result.cause()); + buf.append("\r\n"); + } + + + + @Override + protected TransportEvent getTransportEvent(final byte[] message, final Channel channel) { + return new TransportEvent(message, new NettyHttpLink<>(channel, new ByteEncoder())); + } + + @Override + protected void exceptionCleanup(final ChannelHandlerContext ctx, final Throwable cause) { + // noop + } +} diff --git a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/http/package-info.java b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/http/package-info.java new file mode 100644 index 0000000000..d4589468f4 --- /dev/null +++ b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/http/package-info.java @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +/** + * Wake's remote transportation by HTTP. + */ +package org.apache.reef.wake.remote.transport.netty.http; diff --git a/lang/java/reef-wake/wake/src/test/java/org/apache/reef/wake/test/remote/TransportHttpTest.java b/lang/java/reef-wake/wake/src/test/java/org/apache/reef/wake/test/remote/TransportHttpTest.java new file mode 100644 index 0000000000..fc865ef0a7 --- /dev/null +++ b/lang/java/reef-wake/wake/src/test/java/org/apache/reef/wake/test/remote/TransportHttpTest.java @@ -0,0 +1,162 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.reef.wake.test.remote; + +import org.apache.reef.tang.Injector; +import org.apache.reef.tang.Tang; +import org.apache.reef.tang.exceptions.InjectionException; +import org.apache.reef.wake.EStage; +import org.apache.reef.wake.impl.LoggingUtils; +import org.apache.reef.wake.impl.TimerStage; +import org.apache.reef.wake.remote.Codec; +import org.apache.reef.wake.remote.address.LocalAddressProvider; +import org.apache.reef.wake.remote.impl.ObjectSerializableCodec; +import org.apache.reef.wake.remote.impl.TransportEvent; +import org.apache.reef.wake.remote.transport.Link; +import org.apache.reef.wake.remote.transport.Transport; +import org.apache.reef.wake.remote.transport.TransportFactory; +import org.apache.reef.wake.remote.transport.netty.http.HttpMessagingTransportFactory; +import org.apache.reef.wake.remote.transport.netty.LoggingLinkListener; +import org.apache.reef.wake.test.util.Monitor; +import org.apache.reef.wake.test.util.TimeoutHandler; +import org.junit.Assert; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TestName; + +import java.net.InetSocketAddress; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.logging.Level; + + +/** + * Tests for Transport via http. + */ +public class TransportHttpTest { + private final LocalAddressProvider localAddressProvider; + private final TransportFactory tpFactory; + + public TransportHttpTest() throws InjectionException { + final Injector injector = Tang.Factory.getTang().newInjector(); + this.localAddressProvider = injector.getInstance(LocalAddressProvider.class); + this.tpFactory = injector.getInstance(HttpMessagingTransportFactory.class); + } + + private static final String LOG_PREFIX = "TEST "; + @Rule + public TestName name = new TestName(); + + @Test + public void testHttpTransportString() throws Exception { + System.out.println(LOG_PREFIX + name.getMethodName()); + LoggingUtils.setLoggingLevel(Level.INFO); + + final Monitor monitor = new Monitor(); + final TimerStage timer = new TimerStage(new TimeoutHandler(monitor), 2000, 2000); + + final int expected = 2; + + final String hostAddress = localAddressProvider.getLocalAddress(); + + // Codec + final ReceiverStage stage = + new ReceiverStage<>(new ObjectSerializableCodec(), monitor, expected); + final Transport transport = tpFactory.newInstance(hostAddress, 0, stage, stage, 1, 10000); + final int port = transport.getListeningPort(); + + // sending side + final Link link = transport.open( + new InetSocketAddress(hostAddress, port), + new ObjectSerializableCodec(), + new LoggingLinkListener()); + link.write(new String("hello1")); + link.write(new String("hello2")); + + monitor.mwait(); + transport.close(); + timer.close(); + + Assert.assertEquals(expected, stage.getCount()); + } + + @Test + public void testHttpTransportTestEvent() throws Exception { + System.out.println(LOG_PREFIX + name.getMethodName()); + LoggingUtils.setLoggingLevel(Level.INFO); + + final Monitor monitor = new Monitor(); + final TimerStage timer = new TimerStage(new TimeoutHandler(monitor), 2000, 2000); + + final int expected = 2; + final String hostAddress = localAddressProvider.getLocalAddress(); + + // Codec + final ReceiverStage stage = + new ReceiverStage<>(new ObjectSerializableCodec(), monitor, expected); + final Transport transport = tpFactory.newInstance(hostAddress, 0, stage, stage, 1, 10000); + final int port = transport.getListeningPort(); + + // sending side + final Link link = transport.open( + new InetSocketAddress(hostAddress, port), + new ObjectSerializableCodec(), + new LoggingLinkListener()); + link.write(new TestEvent("hello1", 0.0)); + link.write(new TestEvent("hello2", 1.0)); + + monitor.mwait(); + transport.close(); + timer.close(); + + Assert.assertEquals(expected, stage.getCount()); + } + + class ReceiverStage implements EStage { + + private final Codec codec; + private final Monitor monitor; + private final int expected; + private AtomicInteger count = new AtomicInteger(0); + + ReceiverStage(final Codec codec, final Monitor monitor, final int expected) { + this.codec = codec; + this.monitor = monitor; + this.expected = expected; + } + + int getCount() { + return count.get(); + } + + @Override + public void onNext(final TransportEvent value) { + codec.decode(value.getData()); + + if (count.incrementAndGet() == expected) { + monitor.mnotify(); + } + } + + @Override + public void close() throws Exception { + } + + } + +} From 56d0650d91b3eaa4b6cbe44a57781c2cd8ce9ad4 Mon Sep 17 00:00:00 2001 From: nhne Date: Tue, 25 Jul 2017 14:44:34 +0900 Subject: [PATCH 02/34] Added a test for RemoteManager with Http Transport --- .../test/remote/RemoteManagerTestHttp.java | 489 ++++++++++++++++++ 1 file changed, 489 insertions(+) create mode 100644 lang/java/reef-wake/wake/src/test/java/org/apache/reef/wake/test/remote/RemoteManagerTestHttp.java diff --git a/lang/java/reef-wake/wake/src/test/java/org/apache/reef/wake/test/remote/RemoteManagerTestHttp.java b/lang/java/reef-wake/wake/src/test/java/org/apache/reef/wake/test/remote/RemoteManagerTestHttp.java new file mode 100644 index 0000000000..19ab227845 --- /dev/null +++ b/lang/java/reef-wake/wake/src/test/java/org/apache/reef/wake/test/remote/RemoteManagerTestHttp.java @@ -0,0 +1,489 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.reef.wake.test.remote; + +import org.apache.reef.tang.Injector; +import org.apache.reef.tang.JavaConfigurationBuilder; +import org.apache.reef.tang.Tang; +import org.apache.reef.tang.exceptions.InjectionException; +import org.apache.reef.wake.EventHandler; +import org.apache.reef.wake.impl.LoggingEventHandler; +import org.apache.reef.wake.impl.LoggingUtils; +import org.apache.reef.wake.impl.TimerStage; +import org.apache.reef.wake.remote.*; +import org.apache.reef.wake.remote.address.LocalAddressProvider; +import org.apache.reef.wake.remote.impl.DefaultRemoteIdentifierFactoryImplementation; +import org.apache.reef.wake.remote.impl.MultiCodec; +import org.apache.reef.wake.remote.impl.ObjectSerializableCodec; +import org.apache.reef.wake.remote.ports.TcpPortProvider; +import org.apache.reef.wake.remote.transport.TransportFactory; +import org.apache.reef.wake.remote.transport.netty.http.HttpMessagingTransportFactory; +import org.apache.reef.wake.test.util.Monitor; +import org.apache.reef.wake.test.util.TimeoutHandler; +import org.junit.Assert; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TestName; + +import java.net.UnknownHostException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.logging.Level; + +/** + * Tests for RemoteManagerFactory with HTTP Transport implementation. + */ +public class RemoteManagerTestHttp { + + private final LocalAddressProvider localAddressProvider; + private final RemoteManagerFactory remoteManagerFactory; + + public RemoteManagerTestHttp() throws InjectionException { + final JavaConfigurationBuilder builder = Tang.Factory.getTang().newConfigurationBuilder(); + builder.bindImplementation(TransportFactory.class, HttpMessagingTransportFactory.class); + final Injector injector = Tang.Factory.getTang().newInjector(builder.build()); + this.localAddressProvider = injector.getInstance(LocalAddressProvider.class); + this.remoteManagerFactory = injector.getInstance(RemoteManagerFactory.class); + } + + @Rule + public final TestName name = new TestName(); + + private static final String LOG_PREFIX = "TEST "; + + @Test + public void testRemoteManagerTest() throws Exception { + System.out.println(LOG_PREFIX + name.getMethodName()); + LoggingUtils.setLoggingLevel(Level.INFO); + + final Monitor monitor = new Monitor(); + final TimerStage timer = new TimerStage(new TimeoutHandler(monitor), 2000, 2000); + + final Map, Codec> clazzToCodecMap = new HashMap<>(); + clazzToCodecMap.put(StartEvent.class, new ObjectSerializableCodec()); + clazzToCodecMap.put(TestEvent.class, new ObjectSerializableCodec()); + clazzToCodecMap.put(TestEvent1.class, new ObjectSerializableCodec()); + clazzToCodecMap.put(TestEvent2.class, new ObjectSerializableCodec()); + final Codec codec = new MultiCodec(clazzToCodecMap); + + final String hostAddress = localAddressProvider.getLocalAddress(); + + final RemoteManager rm = this.remoteManagerFactory.getInstance( + "name", hostAddress, 0, codec, new LoggingEventHandler(), false, 3, 10000, + localAddressProvider, Tang.Factory.getTang().newInjector().getInstance(TcpPortProvider.class)); + + final RemoteIdentifier remoteId = rm.getMyIdentifier(); + Assert.assertTrue(rm.getMyIdentifier().equals(remoteId)); + + final EventHandler proxyConnection = rm.getHandler(remoteId, StartEvent.class); + final EventHandler proxyHandler1 = rm.getHandler(remoteId, TestEvent1.class); + final EventHandler proxyHandler2 = rm.getHandler(remoteId, TestEvent2.class); + + final AtomicInteger counter = new AtomicInteger(0); + final int finalSize = 2; + rm.registerHandler(StartEvent.class, new MessageTypeEventHandler(rm, monitor, counter, finalSize)); + + proxyConnection.onNext(new StartEvent()); + + monitor.mwait(); + proxyHandler1.onNext(new TestEvent1("hello1", 0.0)); // registration after send expected to fail + proxyHandler2.onNext(new TestEvent2("hello2", 1.0)); + + monitor.mwait(); + + Assert.assertEquals(finalSize, counter.get()); + + rm.close(); + timer.close(); + } + + @Test + public void testRemoteManagerConnectionRetryTest() throws Exception { + final ExecutorService smExecutor = Executors.newFixedThreadPool(1); + final ExecutorService rmExecutor = Executors.newFixedThreadPool(1); + + final RemoteManager sendingManager = getTestRemoteManager("sender", 9020, 3, 2000); + + final Future smFuture = smExecutor.submit(new SendingRemoteManagerThread(sendingManager, 9010, 20000)); + Thread.sleep(1000); + + final RemoteManager receivingManager = getTestRemoteManager("receiver", 9010, 1, 2000); + final Future rmFuture = rmExecutor.submit(new ReceivingRemoteManagerThread(receivingManager, 20000, 1, 2)); + + final int smCnt = smFuture.get(); + final int rmCnt = rmFuture.get(); + + receivingManager.close(); + sendingManager.close(); + + Assert.assertEquals(0, smCnt); + Assert.assertEquals(2, rmCnt); + } + + @Test + public void testRemoteManagerConnectionRetryWithMultipleSenderTest() throws Exception { + final int numOfSenderThreads = 5; + final ExecutorService smExecutor = Executors.newFixedThreadPool(numOfSenderThreads); + final ExecutorService rmExecutor = Executors.newFixedThreadPool(1); + final ArrayList> smFutures = new ArrayList<>(numOfSenderThreads); + + final RemoteManager sendingManager = getTestRemoteManager("sender", 9030, 3, 5000); + + for (int i = 0; i < numOfSenderThreads; i++) { + smFutures.add(smExecutor.submit(new SendingRemoteManagerThread(sendingManager, 9010, 20000))); + } + + Thread.sleep(2000); + + final RemoteManager receivingManager = getTestRemoteManager("receiver", 9010, 1, 2000); + final Future receivingFuture = + rmExecutor.submit(new ReceivingRemoteManagerThread(receivingManager, 20000, numOfSenderThreads, 2)); + + // waiting sending remote manager. + for (final Future future : smFutures) { + future.get(); + } + + // waiting receiving remote manager + final int rmCnt = receivingFuture.get(); + + sendingManager.close(); + receivingManager.close(); + + // get the result + Assert.assertEquals(2 * numOfSenderThreads, rmCnt); + } + + @Test + public void testRemoteManagerOrderingGuaranteeTest() throws Exception { + System.out.println(LOG_PREFIX + name.getMethodName()); + LoggingUtils.setLoggingLevel(Level.INFO); + + final Monitor monitor = new Monitor(); + final TimerStage timer = new TimerStage(new TimeoutHandler(monitor), 2000, 2000); + + final Map, Codec> clazzToCodecMap = new HashMap<>(); + clazzToCodecMap.put(StartEvent.class, new ObjectSerializableCodec()); + clazzToCodecMap.put(TestEvent.class, new ObjectSerializableCodec()); + clazzToCodecMap.put(TestEvent1.class, new ObjectSerializableCodec()); + clazzToCodecMap.put(TestEvent2.class, new ObjectSerializableCodec()); + final Codec codec = new MultiCodec(clazzToCodecMap); + + final String hostAddress = localAddressProvider.getLocalAddress(); + + final RemoteManager rm = this.remoteManagerFactory.getInstance( + "name", hostAddress, 0, codec, new LoggingEventHandler(), true, 3, 10000, + localAddressProvider, Tang.Factory.getTang().newInjector().getInstance(TcpPortProvider.class)); + + final RemoteIdentifier remoteId = rm.getMyIdentifier(); + + final EventHandler proxyConnection = rm.getHandler(remoteId, StartEvent.class); + final EventHandler proxyHandler1 = rm.getHandler(remoteId, TestEvent1.class); + final EventHandler proxyHandler2 = rm.getHandler(remoteId, TestEvent2.class); + + final AtomicInteger counter = new AtomicInteger(0); + final int finalSize = 2; + rm.registerHandler(StartEvent.class, new MessageTypeEventHandler(rm, monitor, counter, finalSize)); + + proxyConnection.onNext(new StartEvent()); + + monitor.mwait(); + + proxyHandler1.onNext(new TestEvent1("hello1", 0.0)); + proxyHandler2.onNext(new TestEvent2("hello2", 1.0)); + + monitor.mwait(); + + Assert.assertEquals(finalSize, counter.get()); + + rm.close(); + timer.close(); + } + + @Test + public void testRemoteManagerPBufTest() throws Exception { + System.out.println(LOG_PREFIX + name.getMethodName()); + LoggingUtils.setLoggingLevel(Level.INFO); + + final Monitor monitor = new Monitor(); + final TimerStage timer = new TimerStage(new TimeoutHandler(monitor), 2000, 2000); + + final Map, Codec> clazzToCodecMap = new HashMap<>(); + clazzToCodecMap.put(TestEvent.class, new TestEventCodec()); + final Codec codec = new MultiCodec(clazzToCodecMap); + + final String hostAddress = localAddressProvider.getLocalAddress(); + + final RemoteManager rm = this.remoteManagerFactory.getInstance( + "name", hostAddress, 0, codec, new LoggingEventHandler(), false, 3, 10000, + localAddressProvider, Tang.Factory.getTang().newInjector().getInstance(TcpPortProvider.class)); + + final RemoteIdentifier remoteId = rm.getMyIdentifier(); + + final EventHandler proxyHandler = rm.getHandler(remoteId, TestEvent.class); + + final AtomicInteger counter = new AtomicInteger(0); + final int finalSize = 0; + rm.registerHandler(TestEvent.class, new MessageTypeEventHandler(rm, monitor, counter, finalSize)); + + proxyHandler.onNext(new TestEvent("hello", 0.0)); + + monitor.mwait(); + + Assert.assertEquals(finalSize, counter.get()); + + rm.close(); + timer.close(); + } + + @Test + public void testRemoteManagerExceptionTest() { + System.out.println(LOG_PREFIX + name.getMethodName()); + LoggingUtils.setLoggingLevel(Level.INFO); + + final Monitor monitor = new Monitor(); + final TimerStage timer = new TimerStage(new TimeoutHandler(monitor), 2000, 2000); + + final Map, Codec> clazzToCodecMap = new HashMap<>(); + clazzToCodecMap.put(StartEvent.class, new ObjectSerializableCodec()); + clazzToCodecMap.put(TestEvent.class, new ObjectSerializableCodec()); + final Codec codec = new MultiCodec(clazzToCodecMap); + + final ExceptionHandler errorHandler = new ExceptionHandler(monitor); + + try (final RemoteManager rm = remoteManagerFactory.getInstance("name", 0, codec, errorHandler)) { + final RemoteIdentifier remoteId = rm.getMyIdentifier(); + + final EventHandler proxyConnection = rm.getHandler(remoteId, StartEvent.class); + rm.registerHandler(StartEvent.class, new ExceptionGenEventHandler("recvExceptionGen")); + + proxyConnection.onNext(new StartEvent()); + monitor.mwait(); + timer.close(); + + } catch (final UnknownHostException e) { + e.printStackTrace(); + } catch (final Exception e) { + e.printStackTrace(); + } + } + + private RemoteManager getTestRemoteManager(final String rmName, final int localPort, + final int retry, final int retryTimeout) { + final Map, Codec> clazzToCodecMap = new HashMap<>(); + clazzToCodecMap.put(StartEvent.class, new ObjectSerializableCodec()); + clazzToCodecMap.put(TestEvent1.class, new ObjectSerializableCodec()); + clazzToCodecMap.put(TestEvent2.class, new ObjectSerializableCodec()); + final Codec codec = new MultiCodec(clazzToCodecMap); + + final String hostAddress = localAddressProvider.getLocalAddress(); + try { + TcpPortProvider tcpPortProvider = Tang.Factory.getTang().newInjector().getInstance(TcpPortProvider.class); + return remoteManagerFactory.getInstance(rmName, hostAddress, localPort, + codec, new LoggingEventHandler(), false, retry, retryTimeout, + localAddressProvider, tcpPortProvider); + } catch (final InjectionException e) { + throw new RuntimeException(e); + } + } + + private class SendingRemoteManagerThread implements Callable { + + private final int remotePort; + private final int timeout; + private RemoteManager rm; + + SendingRemoteManagerThread(final RemoteManager rm, final int remotePort, final int timeout) { + this.remotePort = remotePort; + this.timeout = timeout; + this.rm = rm; + } + + @Override + public Integer call() throws Exception { + + final Monitor monitor = new Monitor(); + final TimerStage timer = new TimerStage(new TimeoutHandler(monitor), timeout, timeout); + + final String hostAddress = localAddressProvider.getLocalAddress(); + final RemoteIdentifierFactory factory = new DefaultRemoteIdentifierFactoryImplementation(); + final RemoteIdentifier remoteId = factory.getNewInstance("socket://" + hostAddress + ":" + remotePort); + + final EventHandler proxyConnection = rm.getHandler(remoteId, StartEvent.class); + final EventHandler proxyHandler1 = rm.getHandler(remoteId, TestEvent1.class); + final EventHandler proxyHandler2 = rm.getHandler(remoteId, TestEvent2.class); + + final AtomicInteger counter = new AtomicInteger(0); + final int finalSize = 0; + rm.registerHandler(StartEvent.class, new MessageTypeEventHandler(rm, monitor, counter, finalSize)); + + proxyConnection.onNext(new StartEvent()); + + monitor.mwait(); + + proxyHandler1.onNext(new TestEvent1("hello1", 0.0)); // registration after send expected to fail + proxyHandler2.onNext(new TestEvent2("hello2", 0.0)); // registration after send expected to fail + timer.close(); + + return counter.get(); + } + } + + private class ReceivingRemoteManagerThread implements Callable { + + private final int timeout; + private final int numOfConnection; + private final int numOfEvent; + private RemoteManager rm; + + ReceivingRemoteManagerThread(final RemoteManager rm, final int timeout, + final int numOfConnection, final int numOfEvent) { + this.rm = rm; + this.timeout = timeout; + this.numOfConnection = numOfConnection; + this.numOfEvent = numOfEvent; + } + + @Override + public Integer call() throws Exception { + + final Monitor monitor = new Monitor(); + final TimerStage timer = new TimerStage(new TimeoutHandler(monitor), timeout, timeout); + + final AtomicInteger counter = new AtomicInteger(0); + final int finalSize = numOfConnection * numOfEvent; + rm.registerHandler(StartEvent.class, new MessageTypeEventHandler(rm, monitor, counter, finalSize)); + + for (int i = 0; i < numOfConnection; i++) { + monitor.mwait(); + } + monitor.mwait(); + timer.close(); + return counter.get(); + } + } + + class MessageTypeEventHandler implements EventHandler> { + + private final RemoteManager rm; + private final Monitor monitor; + private final AtomicInteger counter; + private final int finalSize; + + MessageTypeEventHandler(final RemoteManager rm, final Monitor monitor, + final AtomicInteger counter, final int finalSize) { + this.rm = rm; + this.monitor = monitor; + this.counter = counter; + this.finalSize = finalSize; + } + + @Override + public void onNext(final RemoteMessage value) { + + final RemoteIdentifier id = value.getIdentifier(); + final T message = value.getMessage(); + + System.out.println(this.getClass() + " " + value + " " + id.toString() + " " + message.toString()); + + System.out.println("Sleeping to force a bug"); + // try { + // Thread.sleep(2000); + //} catch (InterruptedException e) { + + // e.printStackTrace(); + // } + + // register specific handlers + rm.registerHandler(id, TestEvent1.class, + new ConsoleEventHandler("console1", monitor, counter, finalSize)); + rm.registerHandler(id, TestEvent2.class, + new ConsoleEventHandler("console2", monitor, counter, finalSize)); + monitor.mnotify(); + } + } + + class ConsoleEventHandler implements EventHandler { + + private final String name; + private final Monitor monitor; + private final AtomicInteger counter; + private final int finalSize; + + ConsoleEventHandler(final String name, final Monitor monitor, final AtomicInteger counter, final int finalSize) { + this.name = name; + this.monitor = monitor; + this.counter = counter; + this.finalSize = finalSize; + } + + @Override + public void onNext(final T value) { + System.out.println(this.getClass() + " " + name + " " + value); + if (counter.incrementAndGet() == finalSize) { + System.out.println(this.getClass() + " notify counter: " + counter.get()); + monitor.mnotify(); + } + } + } + + class ExceptionGenEventHandler implements EventHandler> { + + private final String name; + + ExceptionGenEventHandler(final String name) { + this.name = name; + } + + @Override + public void onNext(final RemoteMessage value) { + System.out.println(name + " " + value); + throw new TestRuntimeException("Test exception"); + } + } + + class ExceptionHandler implements EventHandler { + + private final Monitor monitor; + + ExceptionHandler(final Monitor monitor) { + this.monitor = monitor; + } + + @Override + public void onNext(final Throwable value) { + System.out.println("!!! ExceptionHandler called : " + value); + monitor.mnotify(); + } + } + + final class TestRuntimeException extends RuntimeException { + private static final long serialVersionUID = 1L; + + TestRuntimeException(final String s) { + super(s); + } + } +} From 49941078fea00aa68bd9fdbdbecf1d81c2e6a2b5 Mon Sep 17 00:00:00 2001 From: Kim Doyoung Date: Mon, 14 Aug 2017 21:22:25 +0900 Subject: [PATCH 03/34] separated NettyChannelFutureListener from NettyLink --- .../netty/NettyChannelFutureListener.java | 43 +++++++++++++++++++ .../remote/transport/netty/NettyLink.java | 20 --------- 2 files changed, 43 insertions(+), 20 deletions(-) create mode 100644 lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyChannelFutureListener.java diff --git a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyChannelFutureListener.java b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyChannelFutureListener.java new file mode 100644 index 0000000000..44f61949cf --- /dev/null +++ b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyChannelFutureListener.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.reef.wake.remote.transport.netty; + +import io.netty.channel.ChannelFuture; +import io.netty.channel.ChannelFutureListener; +import org.apache.reef.wake.remote.transport.LinkListener; + +class NettyChannelFutureListener implements ChannelFutureListener { + + private final T message; + private LinkListener listener; + + NettyChannelFutureListener(final T message, final LinkListener listener) { + this.message = message; + this.listener = listener; + } + + @Override + public void operationComplete(final ChannelFuture channelFuture) throws Exception { + if (channelFuture.isSuccess()) { + listener.onSuccess(message); + } else { + listener.onException(channelFuture.cause(), channelFuture.channel().remoteAddress(), message); + } + } +} diff --git a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyLink.java b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyLink.java index 3c51f2a1c2..df0a8d8f24 100644 --- a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyLink.java +++ b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyLink.java @@ -108,23 +108,3 @@ public String toString() { return "NettyLink: " + channel; // Channel has good .toString() implementation } } - -class NettyChannelFutureListener implements ChannelFutureListener { - - private final T message; - private LinkListener listener; - - NettyChannelFutureListener(final T message, final LinkListener listener) { - this.message = message; - this.listener = listener; - } - - @Override - public void operationComplete(final ChannelFuture channelFuture) throws Exception { - if (channelFuture.isSuccess()) { - listener.onSuccess(message); - } else { - listener.onException(channelFuture.cause(), channelFuture.channel().remoteAddress(), message); - } - } -} From d3f7a466b9be2435067d0bf52ccf021dcea91aa5 Mon Sep 17 00:00:00 2001 From: Kim Doyoung Date: Tue, 15 Aug 2017 00:44:59 +0900 Subject: [PATCH 04/34] Refactored Netty Codes for HTTP Refactored Netty Messaging Transportand Netty Link and Listeners. but still have to change implmentation of MessagingTransportFactory --- .../netty/NettyChannelFutureListener.java | 26 +-- .../netty/NettyChannelInitializer.java | 55 ++++++- .../netty/NettyHttpClientEventListener.java | 99 +++++++++++ .../netty/NettyHttpServerEventListener.java | 155 ++++++++++++++++++ .../remote/transport/netty/NettyLink.java | 50 +++++- .../netty/NettyMessagingTransport.java | 71 ++++++-- 6 files changed, 418 insertions(+), 38 deletions(-) create mode 100644 lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyHttpClientEventListener.java create mode 100644 lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyHttpServerEventListener.java diff --git a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyChannelFutureListener.java b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyChannelFutureListener.java index 44f61949cf..3fefc5a395 100644 --- a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyChannelFutureListener.java +++ b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyChannelFutureListener.java @@ -24,20 +24,20 @@ class NettyChannelFutureListener implements ChannelFutureListener { - private final T message; - private LinkListener listener; + private final T message; + private LinkListener listener; - NettyChannelFutureListener(final T message, final LinkListener listener) { - this.message = message; - this.listener = listener; - } + NettyChannelFutureListener(final T message, final LinkListener listener) { + this.message = message; + this.listener = listener; + } - @Override - public void operationComplete(final ChannelFuture channelFuture) throws Exception { - if (channelFuture.isSuccess()) { - listener.onSuccess(message); - } else { - listener.onException(channelFuture.cause(), channelFuture.channel().remoteAddress(), message); - } + @Override + public void operationComplete(final ChannelFuture channelFuture) throws Exception { + if (channelFuture.isSuccess()) { + listener.onSuccess(message); + } else { + listener.onException(channelFuture.cause(), channelFuture.channel().remoteAddress(), message); } + } } diff --git a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyChannelInitializer.java b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyChannelInitializer.java index 14fd0af17e..80962a2046 100644 --- a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyChannelInitializer.java +++ b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyChannelInitializer.java @@ -24,29 +24,68 @@ import io.netty.handler.codec.LengthFieldPrepender; import io.netty.handler.codec.bytes.ByteArrayDecoder; import io.netty.handler.codec.bytes.ByteArrayEncoder; +import io.netty.handler.codec.http.*; +import io.netty.handler.ssl.SslContext; /** * Netty channel initializer for Transport. */ class NettyChannelInitializer extends ChannelInitializer { + + private static final int HANDLER_NETTY = 100; + private static final int HANDLER_HTTP_SERVER = 101; + private static final int HANDLER_HTTP_CLIENT = 102; + /** * the buffer size of the frame decoder. */ public static final int MAXFRAMELENGTH = 10 * 1024 * 1024; private final NettyChannelHandlerFactory handlerFactory; + private final SslContext sslContext; + private final int type; - NettyChannelInitializer(final NettyChannelHandlerFactory handlerFactory) { + NettyChannelInitializer( + final NettyChannelHandlerFactory handlerFactory, + final SslContext sslContext, + final int type) { this.handlerFactory = handlerFactory; + this.sslContext = sslContext; + this.type = type; } @Override protected void initChannel(final SocketChannel ch) throws Exception { - ch.pipeline() - .addLast("frameDecoder", new LengthFieldBasedFrameDecoder(MAXFRAMELENGTH, 0, 4, 0, 4)) - .addLast("bytesDecoder", new ByteArrayDecoder()) - .addLast("frameEncoder", new LengthFieldPrepender(4)) - .addLast("bytesEncoder", new ByteArrayEncoder()) - .addLast("chunker", new ChunkedReadWriteHandler()) - .addLast("handler", handlerFactory.createChannelInboundHandler()); + switch (this.type) { + case HANDLER_NETTY: + ch.pipeline() + .addLast("frameDecoder", new LengthFieldBasedFrameDecoder(MAXFRAMELENGTH, 0, 4, 0, 4)) + .addLast("bytesDecoder", new ByteArrayDecoder()) + .addLast("frameEncoder", new LengthFieldPrepender(4)) + .addLast("bytesEncoder", new ByteArrayEncoder()) + .addLast("chunker", new ChunkedReadWriteHandler()) + .addLast("handler", handlerFactory.createChannelInboundHandler()); + break; + case HANDLER_HTTP_SERVER: + if (sslContext != null) { + ch.pipeline().addLast(sslContext.newHandler(ch.alloc())); + } + ch.pipeline() + .addLast("codec", new HttpServerCodec()) + .addLast("requestDecoder", new HttpRequestDecoder()) + .addLast("responseEncoder", new HttpResponseEncoder()) + .addLast("handler", handlerFactory.createChannelInboundHandler()); + break; + case HANDLER_HTTP_CLIENT: + if (sslContext != null) { + ch.pipeline().addLast(sslContext.newHandler(ch.alloc())); + } + ch.pipeline() + .addLast("codec", new HttpClientCodec()) + .addLast("decompressor", new HttpContentDecompressor()) + .addLast("handler", handlerFactory.createChannelInboundHandler()); + break; + default: + throw new IllegalArgumentException("Invalid type of channel"); + } } } diff --git a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyHttpClientEventListener.java b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyHttpClientEventListener.java new file mode 100644 index 0000000000..bb10883dde --- /dev/null +++ b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyHttpClientEventListener.java @@ -0,0 +1,99 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.reef.wake.remote.transport.netty; + +import io.netty.buffer.ByteBuf; +import io.netty.channel.Channel; +import io.netty.channel.ChannelHandlerContext; +import io.netty.handler.codec.http.HttpContent; +import io.netty.handler.codec.http.HttpResponse; +import io.netty.handler.codec.http.LastHttpContent; +import io.netty.util.CharsetUtil; +import org.apache.reef.wake.EStage; +import org.apache.reef.wake.remote.impl.TransportEvent; + +import java.net.SocketAddress; +import java.util.concurrent.ConcurrentMap; +import java.util.logging.Level; +import java.util.logging.Logger; + +/** + * A Netty event listener for server. + */ +final class NettyHttpClientEventListener extends AbstractNettyEventListener { + + private static final Logger LOG = Logger.getLogger(NettyHttpClientEventListener.class.getName()); + private StringBuilder buf = new StringBuilder(); + + NettyHttpClientEventListener( + final ConcurrentMap addrToLinkRefMap, + final EStage stage) { + super(addrToLinkRefMap, stage); + } + + @Override + public void channelRead(final ChannelHandlerContext ctx, final Object msg) { + if (msg instanceof HttpResponse) { + System.out.println("CLIENT : IT IS HTTP RESPONSE"); + } else if (msg instanceof HttpContent) { + System.out.println("CLIENT : It is HttpContent"); + HttpContent httpContent = (HttpContent) msg; + ByteBuf content = httpContent.content(); + if (content.isReadable()) { + buf.append("CONTENT: "); + buf.append(content.toString(CharsetUtil.UTF_8)); + buf.append("\r\n"); + } + + if (msg instanceof LastHttpContent) { + final Channel channel = ctx.channel(); + byte[] message = new byte[content.readableBytes()]; + content.readBytes(message); + if (LOG.isLoggable(Level.FINEST)) { + LOG.log(Level.FINEST, "MessageEvent: local: {0} remote: {1} :: {2}", new Object[]{ + channel.localAddress(), channel.remoteAddress(), buf}); + } + + if (message.length > 0) { + // send to the dispatch stage + this.stage.onNext(this.getTransportEvent(message, channel)); + } + } + } else { + LOG.log(Level.SEVERE, "Unknown type of message received: {0}", msg); + } + + } + + @Override + public void channelActive(final ChannelHandlerContext ctx) { + // noop + LOG.log(Level.FINEST, "{0}", ctx); + } + + @Override + protected TransportEvent getTransportEvent(final byte[] message, final Channel channel) { + return new TransportEvent(message, channel.localAddress(), channel.remoteAddress()); + } + + @Override + protected void exceptionCleanup(final ChannelHandlerContext ctx, final Throwable cause) { + this.closeChannel(ctx.channel()); + } +} diff --git a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyHttpServerEventListener.java b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyHttpServerEventListener.java new file mode 100644 index 0000000000..00e6ca638d --- /dev/null +++ b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyHttpServerEventListener.java @@ -0,0 +1,155 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.reef.wake.remote.transport.netty; + +import io.netty.buffer.ByteBuf; +import io.netty.channel.Channel; +import io.netty.channel.ChannelHandlerContext; +import io.netty.handler.codec.DecoderResult; +import io.netty.handler.codec.http.*; +import io.netty.util.CharsetUtil; +import org.apache.reef.wake.EStage; +import org.apache.reef.wake.remote.impl.ByteCodec; +import org.apache.reef.wake.remote.impl.TransportEvent; + +import java.net.SocketAddress; +import java.net.URI; +import java.util.Map; +import java.util.concurrent.ConcurrentMap; +import java.util.logging.Level; + +/** + * A Netty event listener for server side. + */ +final class NettyHttpServerEventListener extends AbstractNettyEventListener { + + private HttpRequest httpRequest; + private final StringBuilder buf = new StringBuilder(); + private final URI uri; + + NettyHttpServerEventListener( + final ConcurrentMap addrToLinkRefMap, + final EStage stage, + final URI uri) { + super(addrToLinkRefMap, stage); + this.uri = uri; + } + + + @Override + public void channelActive(final ChannelHandlerContext ctx) { + final Channel channel = ctx.channel(); + + if (LOG.isLoggable(Level.FINEST)) { + LOG.log(Level.FINEST, "Channel active. key: {0}", channel.remoteAddress()); + } + + this.addrToLinkRefMap.putIfAbsent( + channel.remoteAddress(), new LinkReference(new NettyLink<>( + channel, new ByteCodec(), new LoggingLinkListener(), uri))); + + LOG.log(Level.FINER, "Add connected channel ref: {0}", this.addrToLinkRefMap.get(channel.remoteAddress())); + + } + + @Override + public void channelRead(final ChannelHandlerContext ctx, final Object msg) { + final HttpHeaders headers; + final HttpRequest request; + if(msg instanceof HttpRequest) { + LOG.log(Level.FINEST, "HttpRequest received"); + request = (HttpRequest) msg; + this.httpRequest = request; + headers = request.headers(); + if (!headers.isEmpty()) { + for (Map.Entry h: headers) { + CharSequence key = h.getKey(); + CharSequence value = h.getValue(); + buf.append("HEADER: ").append(key).append(" = ").append(value).append("\r\n"); + } + buf.append("\r\n"); + } + appendDecoderResult(buf, request); + } else if (msg instanceof HttpContent) { + LOG.log(Level.FINEST, "HttpContent received"); + HttpContent httpContent = (HttpContent) msg; + ByteBuf content = httpContent.content(); + if (content.isReadable()) { + buf.append("CONTENT: "); + buf.append(content.toString(CharsetUtil.UTF_8)); + buf.append("\r\n"); + appendDecoderResult(buf, this.httpRequest); + } + + if (msg instanceof LastHttpContent) { + buf.append("END OF CONTENT\r\n"); + LastHttpContent trailer = (LastHttpContent) msg; + if (!trailer.trailingHeaders().isEmpty()) { + buf.append("\r\n"); + for (CharSequence name: trailer.trailingHeaders().names()) { + for (CharSequence value: trailer.trailingHeaders().getAll(name)) { + buf.append("TRAILING HEADER: "); + buf.append(name).append(" = ").append(value).append("\r\n"); + } + } + buf.append("\r\n"); + } + + final Channel channel = ctx.channel(); + byte[] message = new byte[content.readableBytes()]; + content.readBytes(message); + if (LOG.isLoggable(Level.FINEST)) { + LOG.log(Level.FINEST, "MessageEvent: local: {0} remote: {1} :: {2}", new Object[]{ + channel.localAddress(), channel.remoteAddress(), content}); + } + + if (message.length > 0) { + // send to the dispatch stage + this.stage.onNext(this.getTransportEvent(message, channel)); + } + } + } else { + LOG.log(Level.SEVERE, "Unknown type of message received: {0}", msg); + } + LOG.log(Level.FINEST, buf.toString()); + } + + private static void appendDecoderResult(final StringBuilder buf, final HttpObject o) { + DecoderResult result = o.getDecoderResult(); + if (result.isSuccess()) { + return; + } + + buf.append(".. WITH DECODER FAILURE: "); + buf.append(result.cause()); + buf.append("\r\n"); + } + + + + @Override + protected TransportEvent getTransportEvent(final byte[] message, final Channel channel) { + return new TransportEvent(message, new NettyLink<>(channel, new ByteEncoder())); + } + + @Override + protected void exceptionCleanup(final ChannelHandlerContext ctx, final Throwable cause) { + // noop + } +} diff --git a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyLink.java b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyLink.java index df0a8d8f24..a5d7b74785 100644 --- a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyLink.java +++ b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyLink.java @@ -18,15 +18,17 @@ */ package org.apache.reef.wake.remote.transport.netty; +import io.netty.buffer.ByteBuf; import io.netty.buffer.Unpooled; import io.netty.channel.Channel; import io.netty.channel.ChannelFuture; -import io.netty.channel.ChannelFutureListener; +import io.netty.handler.codec.http.*; import org.apache.reef.wake.remote.Encoder; import org.apache.reef.wake.remote.transport.Link; import org.apache.reef.wake.remote.transport.LinkListener; import java.net.SocketAddress; +import java.net.URI; import java.util.logging.Level; import java.util.logging.Logger; @@ -45,6 +47,7 @@ public class NettyLink implements Link { private final Channel channel; private final Encoder encoder; private final LinkListener listener; + private final URI uri; /** * Constructs a link. @@ -64,11 +67,27 @@ public NettyLink(final Channel channel, final Encoder encoder) { * @param listener the link listener */ public NettyLink(final Channel channel, final Encoder encoder, final LinkListener listener) { + this(channel, encoder, listener, null); + } + + /** + * Constructs a link. + * + * @param channel the channel + * @param encoder the encoder + * @param listener the link listener + * @param uri the URI + */ + public NettyLink( + final Channel channel, + final Encoder encoder, + final LinkListener listener, + final URI uri) { this.channel = channel; this.encoder = encoder; this.listener = listener; + this.uri = uri; } - /** * Writes the message to this link. * @@ -77,9 +96,30 @@ public NettyLink(final Channel channel, final Encoder encoder, final @Override public void write(final T message) { LOG.log(Level.FINEST, "write {0} :: {1}", new Object[] {channel, message}); - final ChannelFuture future = channel.writeAndFlush(Unpooled.wrappedBuffer(encoder.encode(message))); - if (listener != null) { - future.addListener(new NettyChannelFutureListener<>(message, listener)); + + if (this.uri != null) { + try { + FullHttpRequest request = new DefaultFullHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.POST, uri.getRawPath()); + request.headers().set(HttpHeaders.Names.HOST, uri.getHost()); + request.headers().set(HttpHeaders.Names.CONNECTION, HttpHeaders.Values.KEEP_ALIVE); + request.headers().set(HttpHeaders.Names.ACCEPT_ENCODING, HttpHeaders.Values.GZIP); + request.headers().set(HttpHeaders.Names.CONTENT_TYPE, "application/wake-transport"); + ByteBuf buf = Unpooled.copiedBuffer(encoder.encode(message)); + request.headers().set(HttpHeaders.Names.CONTENT_LENGTH, buf.readableBytes()); + request.content().clear().writeBytes(buf); + final ChannelFuture future = channel.writeAndFlush(request); + future.sync(); + if (listener != null) { + future.addListener(new NettyChannelFutureListener<>(message, listener)); + } + } catch (InterruptedException ex) { + LOG.log(Level.SEVERE, "Cannot send request to " + uri.getHost(), ex); + } + } else { + final ChannelFuture future = channel.writeAndFlush(Unpooled.wrappedBuffer(encoder.encode(message))); + if (listener != null) { + future.addListener(new NettyChannelFutureListener<>(message, listener)); + } } } diff --git a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyMessagingTransport.java b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyMessagingTransport.java index 2643030116..d7c54151f4 100644 --- a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyMessagingTransport.java +++ b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyMessagingTransport.java @@ -30,6 +30,8 @@ import io.netty.channel.nio.NioEventLoopGroup; import io.netty.channel.socket.nio.NioServerSocketChannel; import io.netty.channel.socket.nio.NioSocketChannel; +import io.netty.handler.ssl.SslContext; +import io.netty.handler.ssl.util.SelfSignedCertificate; import io.netty.util.concurrent.Future; import io.netty.util.concurrent.GlobalEventExecutor; import org.apache.reef.tang.annotations.Parameter; @@ -49,10 +51,7 @@ import javax.inject.Inject; import java.io.IOException; -import java.net.BindException; -import java.net.ConnectException; -import java.net.InetSocketAddress; -import java.net.SocketAddress; +import java.net.*; import java.util.ArrayList; import java.util.Iterator; import java.util.concurrent.ConcurrentHashMap; @@ -62,7 +61,7 @@ import java.util.logging.Logger; /** - * Messaging transport implementation with Netty. + * Messaging transport implementation with Netty and Http. */ public final class NettyMessagingTransport implements Transport { @@ -79,6 +78,13 @@ public final class NettyMessagingTransport implements Transport { private static final int SERVER_WORKER_NUM_THREADS = 20; private static final int CLIENT_WORKER_NUM_THREADS = 10; + private static final int PROTOCOL_NETTY = 100; + private static final int PROTOCOL_HTTP = 101; + + private static final int HANDLER_NETTY = 100; + private static final int HANDLER_HTTP_SERVER = 101; + private static final int HANDLER_HTTP_CLIENT = 102; + private final ConcurrentMap addrToLinkRefMap = new ConcurrentHashMap<>(); private final EventLoopGroup clientWorkerGroup; @@ -95,8 +101,11 @@ public final class NettyMessagingTransport implements Transport { private final int serverPort; private final SocketAddress localAddress; - private final NettyClientEventListener clientEventListener; - private final NettyServerEventListener serverEventListener; + private final AbstractNettyEventListener clientEventListener; + private final AbstractNettyEventListener serverEventListener; + + private final boolean ssl = System.getProperty("ssl") != null; + private final URI uri; private final int numberOfTries; private final int retryTimeout; @@ -122,6 +131,9 @@ private NettyMessagingTransport( @Parameter(RemoteConfiguration.RetryTimeout.class) final int retryTimeout, final TcpPortProvider tcpPortProvider, final LocalAddressProvider localAddressProvider) { + /* TEMPORARY VALUE, NEED TO BE INSERTED INTO PARAMETER */ + /* AFTER TRANSPORTFACTORY IMPLEMENTATION CHANGED */ + final int protocolType = PROTOCOL_NETTY; int p = port; if (p < 0) { @@ -130,10 +142,43 @@ private NettyMessagingTransport( final String host = UNKNOWN_HOST_NAME.equals(hostAddress) ? localAddressProvider.getLocalAddress() : hostAddress; + final SslContext sslContextClient; + final SslContext sslContextServer; + if(protocolType == PROTOCOL_HTTP) { + if (ssl) { + try { + SelfSignedCertificate ssc = new SelfSignedCertificate(); + sslContextClient = SslContext.newServerContext(ssc.certificate(), ssc.privateKey()); + sslContextServer = SslContext.newClientContext(); + LOG.log(Level.FINE, "SSL context created"); + } catch (Exception ex) { + final RuntimeException transportException = + new TransportRuntimeException("Could create SSL Context", ex); + LOG.log(Level.SEVERE, "Cannot create SSL Context", ex); + throw transportException; + } + } else { + LOG.log(Level.FINE, "System property 'ssl' is not set"); + sslContextClient = null; + sslContextServer = null; + } + this.uri = URI.create(ssl ? "https://" : "http://" + hostAddress); + } else { + sslContextClient = null; + sslContextServer = null; + this.uri = null; + } + this.numberOfTries = numberOfTries; this.retryTimeout = retryTimeout; - this.clientEventListener = new NettyClientEventListener(this.addrToLinkRefMap, clientStage); - this.serverEventListener = new NettyServerEventListener(this.addrToLinkRefMap, serverStage); + if (protocolType == PROTOCOL_NETTY) { + this.clientEventListener = new NettyClientEventListener(this.addrToLinkRefMap, clientStage); + this.serverEventListener = new NettyServerEventListener(this.addrToLinkRefMap, serverStage); + } else { + this.clientEventListener = new NettyHttpClientEventListener(this.addrToLinkRefMap, clientStage); + this.serverEventListener = new NettyHttpServerEventListener(this.addrToLinkRefMap, serverStage, this.uri); + } + this.serverBossGroup = new NioEventLoopGroup(SERVER_BOSS_NUM_THREADS, new DefaultThreadFactory(CLASS_NAME + ":ServerBoss")); @@ -146,7 +191,8 @@ private NettyMessagingTransport( this.clientBootstrap.group(this.clientWorkerGroup) .channel(NioSocketChannel.class) .handler(new NettyChannelInitializer(new NettyDefaultChannelHandlerFactory("client", - this.clientChannelGroup, this.clientEventListener))) + this.clientChannelGroup, this.clientEventListener), sslContextClient, + protocolType == PROTOCOL_NETTY ? HANDLER_NETTY : HANDLER_HTTP_CLIENT)) .option(ChannelOption.SO_REUSEADDR, true) .option(ChannelOption.SO_KEEPALIVE, true); @@ -154,7 +200,8 @@ private NettyMessagingTransport( this.serverBootstrap.group(this.serverBossGroup, this.serverWorkerGroup) .channel(NioServerSocketChannel.class) .childHandler(new NettyChannelInitializer(new NettyDefaultChannelHandlerFactory("server", - this.serverChannelGroup, this.serverEventListener))) + this.serverChannelGroup, this.serverEventListener), sslContextServer, + protocolType == PROTOCOL_NETTY ? HANDLER_NETTY : HANDLER_HTTP_SERVER)) .option(ChannelOption.SO_BACKLOG, 128) .option(ChannelOption.SO_REUSEADDR, true) .childOption(ChannelOption.SO_KEEPALIVE, true); @@ -311,7 +358,7 @@ public Link open(final SocketAddress remoteAddr, final Encoder connectFuture = this.clientBootstrap.connect(remoteAddr); connectFuture.syncUninterruptibly(); - link = new NettyLink<>(connectFuture.channel(), encoder, listener); + link = new NettyLink<>(connectFuture.channel(), encoder, listener, this.uri); linkRef.setLink(link); synchronized (flag) { From 1ecda331b50f2ab3a1cae11d7b57f3e418171f97 Mon Sep 17 00:00:00 2001 From: Kim Doyoung Date: Thu, 17 Aug 2017 17:27:49 +0900 Subject: [PATCH 05/34] Removed HttpMessagingTransportFactory and Refactored Code --- .../reef/wake/remote/RemoteConfiguration.java | 15 ++ .../remote/transport/TransportFactory.java | 41 ++++++ .../netty/MessagingTransportFactory.java | 54 +++++++ .../netty/NettyMessagingTransport.java | 13 +- .../http/HttpMessagingTransportFactory.java | 138 ------------------ .../test/remote/RemoteManagerTestHttp.java | 5 +- .../wake/test/remote/TransportHttpTest.java | 8 +- 7 files changed, 123 insertions(+), 151 deletions(-) delete mode 100644 lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/http/HttpMessagingTransportFactory.java diff --git a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/RemoteConfiguration.java b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/RemoteConfiguration.java index 4e3dfa2c4c..9343205828 100644 --- a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/RemoteConfiguration.java +++ b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/RemoteConfiguration.java @@ -47,6 +47,13 @@ public final class RemoteConfiguration { public static final long REMOTE_CONNECTION_RETRY_TIMEOUT = WakeParameters.REMOTE_EXECUTOR_SHUTDOWN_TIMEOUT / (REMOTE_CONNECTION_NUMBER_OF_RETRIES + 1); + + /** + * Unique protocol numbers for choosing protocols. + */ + public static final int PROTOCOL_NETTY = 100; + public static final int PROTOCOL_HTTP = 101; + private RemoteConfiguration() { // empty } @@ -132,4 +139,12 @@ public static final class RemoteClientStage implements Name> { // Intentionally empty } + + /** + * Option for use http. + */ + @NamedParameter(doc = "Option for use http.", default_value = "" + PROTOCOL_NETTY) + public static final class Protocol implements Name { + // Intentionally empty + } } diff --git a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/TransportFactory.java b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/TransportFactory.java index a58da0e274..a2d7eb40be 100644 --- a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/TransportFactory.java +++ b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/TransportFactory.java @@ -62,6 +62,25 @@ Transport newInstance(final String hostAddress, int port, final int numberOfTries, final int retryTimeout); + /** + * Creates a transport. + * + * @param hostAddress a host address + * @param port a listening port + * @param clientStage a transport client-side stage + * @param serverStage a transport server-side stage + * @param numberOfTries the number of retries for connection + * @param retryTimeout retry timeout + * @param protocol protocol to use + * @return transport + */ + Transport newInstance(final String hostAddress, int port, + final EStage clientStage, + final EStage serverStage, + final int numberOfTries, + final int retryTimeout, + final int protocol); + /** * Creates a transport. * @@ -82,5 +101,27 @@ Transport newInstance(final String hostAddress, final int retryTimeout, final TcpPortProvider tcpPortProvider); + /** + * Creates a transport. + * + * @param hostAddress a host address + * @param port a listening port + * @param clientStage a transport client-side stage + * @param serverStage a transport server-side stage + * @param numberOfTries the number of retries for connection + * @param retryTimeout retry timeout + * @param tcpPortProvider tcpPortProvider + * @param protocol protocol to use + * @return transport + */ + Transport newInstance(final String hostAddress, + int port, + final EStage clientStage, + final EStage serverStage, + final int numberOfTries, + final int retryTimeout, + final TcpPortProvider tcpPortProvider, + final int protocol); + } diff --git a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/MessagingTransportFactory.java b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/MessagingTransportFactory.java index 2f0d84d78d..2e9646ec69 100644 --- a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/MessagingTransportFactory.java +++ b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/MessagingTransportFactory.java @@ -101,6 +101,33 @@ public Transport newInstance(final String hostAddress, } } + /** + * Creates a transport. + * + * @param hostAddress a host address + * @param port a listening port + * @param clientStage a client stage + * @param serverStage a server stage + * @param numberOfTries a number of tries + * @param retryTimeout a timeout for retry + */ + @Override + public Transport newInstance(final String hostAddress, + final int port, + final EStage clientStage, + final EStage serverStage, + final int numberOfTries, + final int retryTimeout, + final int protocol) { + try { + TcpPortProvider tcpPortProvider = Tang.Factory.getTang().newInjector().getInstance(TcpPortProvider.class); + return newInstance(hostAddress, port, clientStage, + serverStage, numberOfTries, retryTimeout, tcpPortProvider); + } catch (final InjectionException e) { + throw new RuntimeException(e); + } + } + /** * Creates a transport. * @@ -121,6 +148,32 @@ public Transport newInstance(final String hostAddress, final int retryTimeout, final TcpPortProvider tcpPortProvider) { + return newInstance(hostAddress, port, clientStage, + serverStage, numberOfTries, retryTimeout, tcpPortProvider, RemoteConfiguration.PROTOCOL_NETTY); + } + + /** + * Creates a transport. + * + * @param hostAddress a host address + * @param port a listening port + * @param clientStage a client stage + * @param serverStage a server stage + * @param numberOfTries a number of tries + * @param retryTimeout a timeout for retry + * @param tcpPortProvider a provider for TCP port + * @param protocol a protocol to use + */ + @Override + public Transport newInstance(final String hostAddress, + final int port, + final EStage clientStage, + final EStage serverStage, + final int numberOfTries, + final int retryTimeout, + final TcpPortProvider tcpPortProvider, + final int protocol) { + final Injector injector = Tang.Factory.getTang().newInjector(); injector.bindVolatileParameter(RemoteConfiguration.HostAddress.class, hostAddress); injector.bindVolatileParameter(RemoteConfiguration.Port.class, port); @@ -129,6 +182,7 @@ public Transport newInstance(final String hostAddress, injector.bindVolatileParameter(RemoteConfiguration.NumberOfTries.class, numberOfTries); injector.bindVolatileParameter(RemoteConfiguration.RetryTimeout.class, retryTimeout); injector.bindVolatileInstance(TcpPortProvider.class, tcpPortProvider); + injector.bindVolatileParameter(RemoteConfiguration.Protocol.class, protocol); try { return injector.getInstance(NettyMessagingTransport.class); } catch (final InjectionException e) { diff --git a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyMessagingTransport.java b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyMessagingTransport.java index d7c54151f4..3bc355eee1 100644 --- a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyMessagingTransport.java +++ b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyMessagingTransport.java @@ -60,6 +60,9 @@ import java.util.logging.Level; import java.util.logging.Logger; +import static org.apache.reef.wake.remote.RemoteConfiguration.PROTOCOL_HTTP; +import static org.apache.reef.wake.remote.RemoteConfiguration.PROTOCOL_NETTY; + /** * Messaging transport implementation with Netty and Http. */ @@ -78,9 +81,6 @@ public final class NettyMessagingTransport implements Transport { private static final int SERVER_WORKER_NUM_THREADS = 20; private static final int CLIENT_WORKER_NUM_THREADS = 10; - private static final int PROTOCOL_NETTY = 100; - private static final int PROTOCOL_HTTP = 101; - private static final int HANDLER_NETTY = 100; private static final int HANDLER_HTTP_SERVER = 101; private static final int HANDLER_HTTP_CLIENT = 102; @@ -120,6 +120,7 @@ public final class NettyMessagingTransport implements Transport { * @param numberOfTries the number of tries of connection * @param retryTimeout the timeout of reconnection * @param tcpPortProvider gives an iterator that produces random tcp ports in a range + * @param protocolType the protocol to use for transport */ @Inject private NettyMessagingTransport( @@ -130,10 +131,8 @@ private NettyMessagingTransport( @Parameter(RemoteConfiguration.NumberOfTries.class) final int numberOfTries, @Parameter(RemoteConfiguration.RetryTimeout.class) final int retryTimeout, final TcpPortProvider tcpPortProvider, - final LocalAddressProvider localAddressProvider) { - /* TEMPORARY VALUE, NEED TO BE INSERTED INTO PARAMETER */ - /* AFTER TRANSPORTFACTORY IMPLEMENTATION CHANGED */ - final int protocolType = PROTOCOL_NETTY; + final LocalAddressProvider localAddressProvider, + @Parameter(RemoteConfiguration.Protocol.class) final int protocolType) { int p = port; if (p < 0) { diff --git a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/http/HttpMessagingTransportFactory.java b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/http/HttpMessagingTransportFactory.java deleted file mode 100644 index cf247221d9..0000000000 --- a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/http/HttpMessagingTransportFactory.java +++ /dev/null @@ -1,138 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.reef.wake.remote.transport.netty.http; - -import org.apache.reef.tang.Injector; -import org.apache.reef.tang.Tang; -import org.apache.reef.tang.exceptions.InjectionException; -import org.apache.reef.wake.EStage; -import org.apache.reef.wake.EventHandler; -import org.apache.reef.wake.impl.SyncStage; -import org.apache.reef.wake.remote.RemoteConfiguration; -import org.apache.reef.wake.remote.address.LocalAddressProvider; -import org.apache.reef.wake.remote.impl.TransportEvent; -import org.apache.reef.wake.remote.ports.TcpPortProvider; -import org.apache.reef.wake.remote.transport.Transport; -import org.apache.reef.wake.remote.transport.TransportFactory; - -import javax.inject.Inject; - -/** - * Factory that creates a messaging transport. - */ -public final class HttpMessagingTransportFactory implements TransportFactory { - - private final String localAddress; - - @Inject - private HttpMessagingTransportFactory(final LocalAddressProvider localAddressProvider) { - this.localAddress = localAddressProvider.getLocalAddress(); - } - - /** - * Creates a transport. - * - * @param port a listening port - * @param clientHandler a transport client side handler - * @param serverHandler a transport server side handler - * @param exHandler a exception handler - */ - @Override - public Transport newInstance(final int port, - final EventHandler clientHandler, - final EventHandler serverHandler, - final EventHandler exHandler) { - - final Injector injector = Tang.Factory.getTang().newInjector(); - injector.bindVolatileParameter(RemoteConfiguration.HostAddress.class, this.localAddress); - injector.bindVolatileParameter(RemoteConfiguration.Port.class, port); - injector.bindVolatileParameter(RemoteConfiguration.RemoteClientStage.class, new SyncStage<>(clientHandler)); - injector.bindVolatileParameter(RemoteConfiguration.RemoteServerStage.class, new SyncStage<>(serverHandler)); - - final Transport transport; - try { - transport = injector.getInstance(NettyHttpMessagingTransport.class); - transport.registerErrorHandler(exHandler); - return transport; - } catch (final InjectionException e) { - throw new RuntimeException(e); - } - } - - /** - * Creates a transport. - * - * @param hostAddress a host address - * @param port a listening port - * @param clientStage a client stage - * @param serverStage a server stage - * @param numberOfTries a number of tries - * @param retryTimeout a timeout for retry - */ - @Override - public Transport newInstance(final String hostAddress, - final int port, - final EStage clientStage, - final EStage serverStage, - final int numberOfTries, - final int retryTimeout) { - try { - TcpPortProvider tcpPortProvider = Tang.Factory.getTang().newInjector().getInstance(TcpPortProvider.class); - return newInstance(hostAddress, port, clientStage, - serverStage, numberOfTries, retryTimeout, tcpPortProvider); - } catch (final InjectionException e) { - throw new RuntimeException(e); - } - } - - /** - * Creates a transport. - * - * @param hostAddress a host address - * @param port a listening port - * @param clientStage a client stage - * @param serverStage a server stage - * @param numberOfTries a number of tries - * @param retryTimeout a timeout for retry - * @param tcpPortProvider a provider for TCP port - */ - @Override - public Transport newInstance(final String hostAddress, - final int port, - final EStage clientStage, - final EStage serverStage, - final int numberOfTries, - final int retryTimeout, - final TcpPortProvider tcpPortProvider) { - - final Injector injector = Tang.Factory.getTang().newInjector(); - injector.bindVolatileParameter(RemoteConfiguration.HostAddress.class, hostAddress); - injector.bindVolatileParameter(RemoteConfiguration.Port.class, port); - injector.bindVolatileParameter(RemoteConfiguration.RemoteClientStage.class, clientStage); - injector.bindVolatileParameter(RemoteConfiguration.RemoteServerStage.class, serverStage); - injector.bindVolatileParameter(RemoteConfiguration.NumberOfTries.class, numberOfTries); - injector.bindVolatileParameter(RemoteConfiguration.RetryTimeout.class, retryTimeout); - injector.bindVolatileInstance(TcpPortProvider.class, tcpPortProvider); - try { - return injector.getInstance(NettyHttpMessagingTransport.class); - } catch (final InjectionException e) { - throw new RuntimeException(e); - } - } -} diff --git a/lang/java/reef-wake/wake/src/test/java/org/apache/reef/wake/test/remote/RemoteManagerTestHttp.java b/lang/java/reef-wake/wake/src/test/java/org/apache/reef/wake/test/remote/RemoteManagerTestHttp.java index 19ab227845..f506016fec 100644 --- a/lang/java/reef-wake/wake/src/test/java/org/apache/reef/wake/test/remote/RemoteManagerTestHttp.java +++ b/lang/java/reef-wake/wake/src/test/java/org/apache/reef/wake/test/remote/RemoteManagerTestHttp.java @@ -33,7 +33,7 @@ import org.apache.reef.wake.remote.impl.ObjectSerializableCodec; import org.apache.reef.wake.remote.ports.TcpPortProvider; import org.apache.reef.wake.remote.transport.TransportFactory; -import org.apache.reef.wake.remote.transport.netty.http.HttpMessagingTransportFactory; +import org.apache.reef.wake.remote.transport.netty.MessagingTransportFactory; import org.apache.reef.wake.test.util.Monitor; import org.apache.reef.wake.test.util.TimeoutHandler; import org.junit.Assert; @@ -62,7 +62,8 @@ public class RemoteManagerTestHttp { public RemoteManagerTestHttp() throws InjectionException { final JavaConfigurationBuilder builder = Tang.Factory.getTang().newConfigurationBuilder(); - builder.bindImplementation(TransportFactory.class, HttpMessagingTransportFactory.class); + builder.bindImplementation(TransportFactory.class, MessagingTransportFactory.class); + builder.bindNamedParameter(RemoteConfiguration.Protocol.class, "101"); final Injector injector = Tang.Factory.getTang().newInjector(builder.build()); this.localAddressProvider = injector.getInstance(LocalAddressProvider.class); this.remoteManagerFactory = injector.getInstance(RemoteManagerFactory.class); diff --git a/lang/java/reef-wake/wake/src/test/java/org/apache/reef/wake/test/remote/TransportHttpTest.java b/lang/java/reef-wake/wake/src/test/java/org/apache/reef/wake/test/remote/TransportHttpTest.java index fc865ef0a7..ff20ce0dba 100644 --- a/lang/java/reef-wake/wake/src/test/java/org/apache/reef/wake/test/remote/TransportHttpTest.java +++ b/lang/java/reef-wake/wake/src/test/java/org/apache/reef/wake/test/remote/TransportHttpTest.java @@ -31,7 +31,7 @@ import org.apache.reef.wake.remote.transport.Link; import org.apache.reef.wake.remote.transport.Transport; import org.apache.reef.wake.remote.transport.TransportFactory; -import org.apache.reef.wake.remote.transport.netty.http.HttpMessagingTransportFactory; +import org.apache.reef.wake.remote.transport.netty.MessagingTransportFactory; import org.apache.reef.wake.remote.transport.netty.LoggingLinkListener; import org.apache.reef.wake.test.util.Monitor; import org.apache.reef.wake.test.util.TimeoutHandler; @@ -55,7 +55,7 @@ public class TransportHttpTest { public TransportHttpTest() throws InjectionException { final Injector injector = Tang.Factory.getTang().newInjector(); this.localAddressProvider = injector.getInstance(LocalAddressProvider.class); - this.tpFactory = injector.getInstance(HttpMessagingTransportFactory.class); + this.tpFactory = injector.getInstance(MessagingTransportFactory.class); } private static final String LOG_PREFIX = "TEST "; @@ -77,7 +77,7 @@ public void testHttpTransportString() throws Exception { // Codec final ReceiverStage stage = new ReceiverStage<>(new ObjectSerializableCodec(), monitor, expected); - final Transport transport = tpFactory.newInstance(hostAddress, 0, stage, stage, 1, 10000); + final Transport transport = tpFactory.newInstance(hostAddress, 0, stage, stage, 1, 10000, 101); final int port = transport.getListeningPort(); // sending side @@ -109,7 +109,7 @@ public void testHttpTransportTestEvent() throws Exception { // Codec final ReceiverStage stage = new ReceiverStage<>(new ObjectSerializableCodec(), monitor, expected); - final Transport transport = tpFactory.newInstance(hostAddress, 0, stage, stage, 1, 10000); + final Transport transport = tpFactory.newInstance(hostAddress, 0, stage, stage, 1, 10000, 101); final int port = transport.getListeningPort(); // sending side From 9510d614f28418c2dabcc3df814a41f6de182081 Mon Sep 17 00:00:00 2001 From: Kim Doyoung Date: Thu, 17 Aug 2017 18:05:11 +0900 Subject: [PATCH 06/34] Removed package reef.wake.remote.transport.netty.http --- .../netty/http/NettyHttpChannelHandler.java | 128 ------ .../http/NettyHttpChannelHandlerFactory.java | 51 --- .../http/NettyHttpChannelInitializer.java | 70 --- .../http/NettyHttpClientEventListener.java | 101 ----- .../transport/netty/http/NettyHttpLink.java | 154 ------- .../http/NettyHttpMessagingTransport.java | 415 ------------------ .../http/NettyHttpServerEventListener.java | 159 ------- .../transport/netty/http/package-info.java | 22 - 8 files changed, 1100 deletions(-) delete mode 100644 lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/http/NettyHttpChannelHandler.java delete mode 100644 lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/http/NettyHttpChannelHandlerFactory.java delete mode 100644 lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/http/NettyHttpChannelInitializer.java delete mode 100644 lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/http/NettyHttpClientEventListener.java delete mode 100644 lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/http/NettyHttpLink.java delete mode 100644 lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/http/NettyHttpMessagingTransport.java delete mode 100644 lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/http/NettyHttpServerEventListener.java delete mode 100644 lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/http/package-info.java diff --git a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/http/NettyHttpChannelHandler.java b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/http/NettyHttpChannelHandler.java deleted file mode 100644 index 7af5a60745..0000000000 --- a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/http/NettyHttpChannelHandler.java +++ /dev/null @@ -1,128 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.reef.wake.remote.transport.netty.http; - -import io.netty.channel.Channel; -import io.netty.channel.ChannelHandlerContext; -import io.netty.channel.ChannelInboundHandlerAdapter; -import io.netty.channel.group.ChannelGroup; -import io.netty.util.ReferenceCountUtil; -import org.apache.reef.wake.remote.transport.netty.NettyEventListener; - -import java.util.logging.Level; -import java.util.logging.Logger; - -/** - * Netty channel handler for channel status(active/inactive) and incoming data. - */ -final class NettyHttpChannelHandler extends ChannelInboundHandlerAdapter { - - private static final Logger LOG = Logger.getLogger(NettyHttpChannelHandler.class.getName()); - - private final String tag; - private final ChannelGroup channelGroup; - private final NettyEventListener listener; - - /** - * Constructs a Netty channel handler. - * - * @param tag tag string - * @param channelGroup the channel group - * @param listener the Netty event listener - */ - NettyHttpChannelHandler( - final String tag, final ChannelGroup channelGroup, final NettyEventListener listener) { - this.tag = tag; - this.channelGroup = channelGroup; - this.listener = listener; - } - - /** - * Handle the incoming message: pass it to the listener. - * - * @param ctx the context object for this handler. - * @param msg the message. - * @throws Exception - */ - @Override - public void channelRead( - final ChannelHandlerContext ctx, final Object msg) throws Exception { - try { - this.listener.channelRead(ctx, msg); - } finally { - ReferenceCountUtil.release(msg); - } - } - - /** - * Flushes after channel read complete. - * - * @param ctx - * @throws Exception - */ - - @Override - public void channelReadComplete(final ChannelHandlerContext ctx) throws Exception { - ctx.flush(); - } - - /** - * Handles the channel active event. - * - * @param ctx the context object for this handler - * @throws Exception - */ - @Override - public void channelActive(final ChannelHandlerContext ctx) throws Exception { - this.channelGroup.add(ctx.channel()); - this.listener.channelActive(ctx); - super.channelActive(ctx); - } - - /** - * Handles the channel inactive event. - * - * @param ctx the context object for this handler - * @throws Exception - */ - @Override - public void channelInactive(final ChannelHandlerContext ctx) throws Exception { - this.listener.channelInactive(ctx); - super.channelInactive(ctx); - } - - /** - * Handles the exception event. - * - * @param ctx the context object for this handler - * @param cause the cause - * @throws Exception - */ - @Override - public void exceptionCaught(final ChannelHandlerContext ctx, final Throwable cause) { - final Channel channel = ctx.channel(); - - LOG.log(Level.INFO, "Unexpected exception from downstream. channel: {0} local: {1} remote: {2}", - new Object[]{channel, channel.localAddress(), channel.remoteAddress()}); - LOG.log(Level.WARNING, "Unexpected exception from downstream.", cause); - channel.close(); - this.listener.exceptionCaught(ctx, cause); - } - -} diff --git a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/http/NettyHttpChannelHandlerFactory.java b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/http/NettyHttpChannelHandlerFactory.java deleted file mode 100644 index 5df6cee35e..0000000000 --- a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/http/NettyHttpChannelHandlerFactory.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.reef.wake.remote.transport.netty.http; - -import io.netty.channel.ChannelInboundHandlerAdapter; -import io.netty.channel.group.ChannelGroup; -import org.apache.reef.wake.remote.transport.netty.NettyChannelHandlerFactory; -import org.apache.reef.wake.remote.transport.netty.NettyEventListener; - -/** - * Default Netty channel handler factory. - */ -final class NettyHttpChannelHandlerFactory implements NettyChannelHandlerFactory { - - private final String tag; - private final ChannelGroup group; - private final NettyEventListener listener; - - NettyHttpChannelHandlerFactory( - final String tag, final ChannelGroup group, final NettyEventListener listener) { - this.tag = tag; - this.group = group; - this.listener = listener; - } - - /** - * Creates a Netty channel handler. - * - * @return a simple channel upstream handler. - */ - @Override - public ChannelInboundHandlerAdapter createChannelInboundHandler() { - return new NettyHttpChannelHandler(this.tag, this.group, this.listener); - } -} diff --git a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/http/NettyHttpChannelInitializer.java b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/http/NettyHttpChannelInitializer.java deleted file mode 100644 index c578ae6432..0000000000 --- a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/http/NettyHttpChannelInitializer.java +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.reef.wake.remote.transport.netty.http; - -import io.netty.channel.ChannelInitializer; -import io.netty.channel.socket.SocketChannel; -import io.netty.handler.codec.http.*; -import io.netty.handler.ssl.SslContext; -import org.apache.reef.wake.remote.transport.netty.NettyChannelHandlerFactory; - -/** - * Netty Http channel initializer for Transport. - */ -class NettyHttpChannelInitializer extends ChannelInitializer { - - static final int TYPE_SERVER = 0; - static final int TYPE_CLIENT = 1; - - private final NettyChannelHandlerFactory handlerFactory; - private final SslContext sslContext; - private final int type; - - NettyHttpChannelInitializer( - final NettyChannelHandlerFactory handlerFactory, - final SslContext sslContext, - final int type) { - if(type != TYPE_SERVER && type != TYPE_CLIENT){ - throw new IllegalArgumentException("Invalid type of channel"); - } - this.handlerFactory = handlerFactory; - this.sslContext = sslContext; - this.type = type; - } - - @Override - protected void initChannel(final SocketChannel ch) throws Exception { - if(sslContext != null) { - ch.pipeline().addLast(sslContext.newHandler(ch.alloc())); - } - if(type == TYPE_SERVER) { - // Init channel for server - ch.pipeline() - .addLast("codec", new HttpServerCodec()) - .addLast("requestDecoder", new HttpRequestDecoder()) - .addLast("responseEncoder", new HttpResponseEncoder()); - } else if (type == TYPE_CLIENT) { - // Init channel for client - ch.pipeline() - .addLast("codec", new HttpClientCodec()) - .addLast("decompressor", new HttpContentDecompressor()); - } - ch.pipeline().addLast("handler", handlerFactory.createChannelInboundHandler()); - } -} diff --git a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/http/NettyHttpClientEventListener.java b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/http/NettyHttpClientEventListener.java deleted file mode 100644 index a8c822b951..0000000000 --- a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/http/NettyHttpClientEventListener.java +++ /dev/null @@ -1,101 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.reef.wake.remote.transport.netty.http; - -import io.netty.buffer.ByteBuf; -import io.netty.channel.Channel; -import io.netty.channel.ChannelHandlerContext; -import io.netty.handler.codec.http.HttpContent; -import io.netty.handler.codec.http.HttpResponse; -import io.netty.handler.codec.http.LastHttpContent; -import io.netty.util.CharsetUtil; -import org.apache.reef.wake.EStage; -import org.apache.reef.wake.remote.impl.TransportEvent; -import org.apache.reef.wake.remote.transport.netty.AbstractNettyEventListener; -import org.apache.reef.wake.remote.transport.netty.LinkReference; - -import java.net.SocketAddress; -import java.util.concurrent.ConcurrentMap; -import java.util.logging.Level; -import java.util.logging.Logger; - -/** - * A Netty event listener for server. - */ -final class NettyHttpClientEventListener extends AbstractNettyEventListener { - - private static final Logger LOG = Logger.getLogger(NettyHttpClientEventListener.class.getName()); - private StringBuilder buf = new StringBuilder(); - - NettyHttpClientEventListener( - final ConcurrentMap addrToLinkRefMap, - final EStage stage) { - super(addrToLinkRefMap, stage); - } - - @Override - public void channelRead(final ChannelHandlerContext ctx, final Object msg) { - if (msg instanceof HttpResponse) { - System.out.println("CLIENT : IT IS HTTP RESPONSE"); - } else if (msg instanceof HttpContent) { - System.out.println("CLIENT : It is HttpContent"); - HttpContent httpContent = (HttpContent) msg; - ByteBuf content = httpContent.content(); - if (content.isReadable()) { - buf.append("CONTENT: "); - buf.append(content.toString(CharsetUtil.UTF_8)); - buf.append("\r\n"); - } - - if (msg instanceof LastHttpContent) { - final Channel channel = ctx.channel(); - byte[] message = new byte[content.readableBytes()]; - content.readBytes(message); - if (LOG.isLoggable(Level.FINEST)) { - LOG.log(Level.FINEST, "MessageEvent: local: {0} remote: {1} :: {2}", new Object[]{ - channel.localAddress(), channel.remoteAddress(), buf}); - } - - if (message.length > 0) { - // send to the dispatch stage - this.stage.onNext(this.getTransportEvent(message, channel)); - } - } - } else { - LOG.log(Level.SEVERE, "Unknown type of message received: {0}", msg); - } - - } - - @Override - public void channelActive(final ChannelHandlerContext ctx) { - // noop - LOG.log(Level.FINEST, "{0}", ctx); - } - - @Override - protected TransportEvent getTransportEvent(final byte[] message, final Channel channel) { - return new TransportEvent(message, channel.localAddress(), channel.remoteAddress()); - } - - @Override - protected void exceptionCleanup(final ChannelHandlerContext ctx, final Throwable cause) { - this.closeChannel(ctx.channel()); - } -} diff --git a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/http/NettyHttpLink.java b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/http/NettyHttpLink.java deleted file mode 100644 index 25ce9ddbb4..0000000000 --- a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/http/NettyHttpLink.java +++ /dev/null @@ -1,154 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.reef.wake.remote.transport.netty.http; - -import io.netty.buffer.ByteBuf; -import io.netty.buffer.Unpooled; -import io.netty.channel.Channel; -import io.netty.channel.ChannelFuture; -import io.netty.channel.ChannelFutureListener; -import io.netty.handler.codec.http.*; -import org.apache.reef.wake.remote.Encoder; -import org.apache.reef.wake.remote.transport.Link; -import org.apache.reef.wake.remote.transport.LinkListener; - -import java.net.SocketAddress; -import java.net.URI; -import java.util.logging.Level; -import java.util.logging.Logger; - -/** - * Link implementation with Netty Http. - * - * If you set a {@code LinkListener}, it keeps message until writeAndFlush operation completes - * and notifies whether the sent message transferred successfully through the listener. - */ -public class NettyHttpLink implements Link { - - public static final int INT_SIZE = Integer.SIZE / Byte.SIZE; - - private static final Logger LOG = Logger.getLogger(NettyHttpLink.class.getName()); - - private final Channel channel; - private final Encoder encoder; - private final LinkListener listener; - private final URI uri; - - /** - * Constructs a link. - * - * @param channel the channel - * @param encoder the encoder - */ - public NettyHttpLink(final Channel channel, final Encoder encoder) { - this(channel, encoder, null, URI.create("http://127.0.0.1")); - } - - /** - * Constructs a link. - * - * @param channel the channel - * @param encoder the encoder - * @param listener the link listener - * @param uri the URI - */ - public NettyHttpLink( - final Channel channel, - final Encoder encoder, - final LinkListener listener, - final URI uri) { - this.channel = channel; - this.encoder = encoder; - this.listener = listener; - this.uri = uri; - } - - /** - * Writes the message to this link via HTTP. - * - * @param message the message - */ - @Override - public void write(final T message) { - - try { - LOG.log(Level.FINE, "write {0} :: {1}", new Object[] {channel, message}); - FullHttpRequest request = new DefaultFullHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.POST, uri.getRawPath()); - request.headers().set(HttpHeaders.Names.HOST, uri.getHost()); - request.headers().set(HttpHeaders.Names.CONNECTION, HttpHeaders.Values.KEEP_ALIVE); - request.headers().set(HttpHeaders.Names.ACCEPT_ENCODING, HttpHeaders.Values.GZIP); - request.headers().set(HttpHeaders.Names.CONTENT_TYPE, "application/wake-transport"); - ByteBuf buf = Unpooled.copiedBuffer(encoder.encode(message)); - request.headers().set(HttpHeaders.Names.CONTENT_LENGTH, buf.readableBytes()); - request.content().clear().writeBytes(buf); - final ChannelFuture future = channel.writeAndFlush(request); - future.sync(); - if (listener != null) { - future.addListener(new NettyHttpChannelFutureListener<>(message, listener)); - } - } catch (InterruptedException ex) { - LOG.log(Level.SEVERE, "Cannot send request to " + uri.getHost(), ex); - } - } - - /** - * Gets a local address of the link. - * - * @return a local socket address - */ - @Override - public SocketAddress getLocalAddress() { - return channel.localAddress(); - } - - /** - * Gets a remote address of the link. - * - * @return a remote socket address - */ - @Override - public SocketAddress getRemoteAddress() { - return channel.remoteAddress(); - } - - @Override - public String toString() { - return "NettyHttpLink: " + channel; // Channel has good .toString() implementation - } -} - -class NettyHttpChannelFutureListener implements ChannelFutureListener { - - private final T message; - private LinkListener listener; - - NettyHttpChannelFutureListener(final T message, final LinkListener listener) { - this.message = message; - this.listener = listener; - } - - @Override - public void operationComplete(final ChannelFuture channelFuture) throws Exception { - if (channelFuture.isSuccess()) { - listener.onSuccess(message); - } else { - listener.onException(channelFuture.cause(), channelFuture.channel().remoteAddress(), message); - } - } -} diff --git a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/http/NettyHttpMessagingTransport.java b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/http/NettyHttpMessagingTransport.java deleted file mode 100644 index da116c27d6..0000000000 --- a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/http/NettyHttpMessagingTransport.java +++ /dev/null @@ -1,415 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.reef.wake.remote.transport.netty.http; - -import io.netty.bootstrap.Bootstrap; -import io.netty.bootstrap.ServerBootstrap; -import io.netty.channel.Channel; -import io.netty.channel.ChannelFuture; -import io.netty.channel.ChannelOption; -import io.netty.channel.EventLoopGroup; -import io.netty.channel.group.ChannelGroup; -import io.netty.channel.group.ChannelGroupFuture; -import io.netty.channel.group.DefaultChannelGroup; -import io.netty.channel.nio.NioEventLoopGroup; -import io.netty.channel.socket.nio.NioServerSocketChannel; -import io.netty.channel.socket.nio.NioSocketChannel; -import io.netty.handler.ssl.SslContext; -import io.netty.handler.ssl.util.SelfSignedCertificate; -import io.netty.util.concurrent.Future; -import io.netty.util.concurrent.GlobalEventExecutor; -import org.apache.reef.tang.annotations.Parameter; -import org.apache.reef.wake.EStage; -import org.apache.reef.wake.EventHandler; -import org.apache.reef.wake.impl.DefaultThreadFactory; -import org.apache.reef.wake.remote.Encoder; -import org.apache.reef.wake.remote.RemoteConfiguration; -import org.apache.reef.wake.remote.address.LocalAddressProvider; -import org.apache.reef.wake.remote.exception.RemoteRuntimeException; -import org.apache.reef.wake.remote.impl.TransportEvent; -import org.apache.reef.wake.remote.ports.TcpPortProvider; -import org.apache.reef.wake.remote.transport.Link; -import org.apache.reef.wake.remote.transport.LinkListener; -import org.apache.reef.wake.remote.transport.Transport; -import org.apache.reef.wake.remote.transport.exception.TransportRuntimeException; -import org.apache.reef.wake.remote.transport.netty.LinkReference; - -import javax.inject.Inject; -import java.io.IOException; -import java.net.*; -import java.util.ArrayList; -import java.util.Iterator; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.logging.Level; -import java.util.logging.Logger; - -import static org.apache.reef.wake.remote.transport.netty.http.NettyHttpChannelInitializer.TYPE_CLIENT; -import static org.apache.reef.wake.remote.transport.netty.http.NettyHttpChannelInitializer.TYPE_SERVER; - -/** - * Messaging transport implementation with Netty HTTP. - */ -public final class NettyHttpMessagingTransport implements Transport { - - /** - * Indicates a hostname that isn't set or known. - */ - public static final String UNKNOWN_HOST_NAME = "##UNKNOWN##"; - - private static final String CLASS_NAME = NettyHttpMessagingTransport.class.getSimpleName(); - - private static final Logger LOG = Logger.getLogger(CLASS_NAME); - - private static final int SERVER_BOSS_NUM_THREADS = 3; - private static final int SERVER_WORKER_NUM_THREADS = 20; - private static final int CLIENT_WORKER_NUM_THREADS = 10; - - private final ConcurrentMap addrToLinkRefMap = new ConcurrentHashMap<>(); - - private final EventLoopGroup clientWorkerGroup; - private final EventLoopGroup serverBossGroup; - private final EventLoopGroup serverWorkerGroup; - - private final Bootstrap clientBootstrap; - private final ServerBootstrap serverBootstrap; - private final Channel acceptor; - - private final ChannelGroup clientChannelGroup = new DefaultChannelGroup(GlobalEventExecutor.INSTANCE); - private final ChannelGroup serverChannelGroup = new DefaultChannelGroup(GlobalEventExecutor.INSTANCE); - - private final int serverPort; - private final SocketAddress localAddress; - - private final boolean ssl = System.getProperty("ssl") != null; - private final URI uri; - - private final NettyHttpClientEventListener clientEventListener; - private final NettyHttpServerEventListener serverEventListener; - - private final int numberOfTries; - private final int retryTimeout; - - /** - * Constructs a messaging transport. - * - * @param hostAddress the server host address - * @param port the server listening port; when it is 0, randomly assign a port number - * @param clientStage the client-side stage that handles transport events - * @param serverStage the server-side stage that handles transport events - * @param numberOfTries the number of tries of connection - * @param retryTimeout the timeout of reconnection - * @param tcpPortProvider gives an iterator that produces random tcp ports in a range - */ - @Inject - private NettyHttpMessagingTransport( - @Parameter(RemoteConfiguration.HostAddress.class) final String hostAddress, - @Parameter(RemoteConfiguration.Port.class) final int port, - @Parameter(RemoteConfiguration.RemoteClientStage.class) final EStage clientStage, - @Parameter(RemoteConfiguration.RemoteServerStage.class) final EStage serverStage, - @Parameter(RemoteConfiguration.NumberOfTries.class) final int numberOfTries, - @Parameter(RemoteConfiguration.RetryTimeout.class) final int retryTimeout, - final TcpPortProvider tcpPortProvider, - final LocalAddressProvider localAddressProvider) { - - int p = port; - if (p < 0) { - throw new RemoteRuntimeException("Invalid server port: " + p); - } - - final String host = UNKNOWN_HOST_NAME.equals(hostAddress) ? localAddressProvider.getLocalAddress() : hostAddress; - - final SslContext sslContextClient; - final SslContext sslContextServer; - if(ssl){ - try { - SelfSignedCertificate ssc = new SelfSignedCertificate(); - sslContextClient = SslContext.newServerContext(ssc.certificate(), ssc.privateKey()); - sslContextServer = SslContext.newClientContext(); - LOG.log(Level.FINE, "SSL context created"); - } catch (Exception ex) { - final RuntimeException transportException = - new TransportRuntimeException("Could create SSL Context", ex); - LOG.log(Level.SEVERE, "Cannot create SSL Context", ex); - throw transportException; - } - } else { - LOG.log(Level.FINE, "System property 'ssl' is not set"); - sslContextClient = null; - sslContextServer = null; - } - - this.uri = URI.create(ssl ? "https://" : "http://" + hostAddress); - - this.numberOfTries = numberOfTries; - this.retryTimeout = retryTimeout; - this.clientEventListener = new NettyHttpClientEventListener(this.addrToLinkRefMap, clientStage); - this.serverEventListener = new NettyHttpServerEventListener(this.addrToLinkRefMap, serverStage, this.uri); - - this.serverBossGroup = new NioEventLoopGroup(SERVER_BOSS_NUM_THREADS, - new DefaultThreadFactory(CLASS_NAME + ":ServerBoss")); - this.serverWorkerGroup = new NioEventLoopGroup(SERVER_WORKER_NUM_THREADS, - new DefaultThreadFactory(CLASS_NAME + ":ServerWorker")); - this.clientWorkerGroup = new NioEventLoopGroup(CLIENT_WORKER_NUM_THREADS, - new DefaultThreadFactory(CLASS_NAME + ":ClientWorker")); - - this.clientBootstrap = new Bootstrap(); - this.clientBootstrap.group(this.clientWorkerGroup) - .channel(NioSocketChannel.class) - .handler(new NettyHttpChannelInitializer(new NettyHttpChannelHandlerFactory("client", - this.clientChannelGroup, this.clientEventListener), sslContextClient, TYPE_CLIENT)) - .option(ChannelOption.SO_REUSEADDR, true) - .option(ChannelOption.SO_KEEPALIVE, true); - - this.serverBootstrap = new ServerBootstrap(); - this.serverBootstrap.group(this.serverBossGroup, this.serverWorkerGroup) - .channel(NioServerSocketChannel.class) - .childHandler(new NettyHttpChannelInitializer(new NettyHttpChannelHandlerFactory("server", - this.serverChannelGroup, this.serverEventListener), sslContextServer, TYPE_SERVER)) - .option(ChannelOption.SO_BACKLOG, 128) - .option(ChannelOption.SO_REUSEADDR, true) - .childOption(ChannelOption.SO_KEEPALIVE, true); - - LOG.log(Level.FINE, "Binding to {0}", p); - - Channel acceptorFound = null; - try { - if (p > 0) { - acceptorFound = this.serverBootstrap.bind(new InetSocketAddress(host, p)).sync().channel(); - } else { - final Iterator ports = tcpPortProvider.iterator(); - while (acceptorFound == null) { - if (!ports.hasNext()) { - throw new IllegalStateException("tcpPortProvider cannot find a free port."); - } - p = ports.next(); - LOG.log(Level.FINEST, "Try port {0}", p); - try { - acceptorFound = this.serverBootstrap.bind(new InetSocketAddress(host, p)).sync().channel(); - } catch (final Exception ex) { - if (ex instanceof BindException) { - LOG.log(Level.FINEST, "The port {0} is already bound. Try again", p); - } else { - throw ex; - } - } - } - } - } catch (final IllegalStateException ex) { - final RuntimeException transportException = - new TransportRuntimeException("tcpPortProvider failed to return free ports.", ex); - LOG.log(Level.SEVERE, "Cannot find a free port with " + tcpPortProvider, transportException); - - this.clientWorkerGroup.shutdownGracefully(); - this.serverBossGroup.shutdownGracefully(); - this.serverWorkerGroup.shutdownGracefully(); - throw transportException; - - } catch (final Exception ex) { - final RuntimeException transportException = - new TransportRuntimeException("Cannot bind to port " + p, ex); - LOG.log(Level.SEVERE, "Cannot bind to port " + p, ex); - - this.clientWorkerGroup.shutdownGracefully(); - this.serverBossGroup.shutdownGracefully(); - this.serverWorkerGroup.shutdownGracefully(); - throw transportException; - } - this.acceptor = acceptorFound; - this.serverPort = p; - this.localAddress = new InetSocketAddress(host, this.serverPort); - - LOG.log(Level.FINE, "Starting netty transport socket address: {0}", this.localAddress); - } - - /** - * Closes all channels and releases all resources. - */ - @Override - public void close() { - - LOG.log(Level.FINE, "Closing netty transport socket address: {0}", this.localAddress); - - final ChannelGroupFuture clientChannelGroupFuture = this.clientChannelGroup.close(); - final ChannelGroupFuture serverChannelGroupFuture = this.serverChannelGroup.close(); - final ChannelFuture acceptorFuture = this.acceptor.close(); - - final ArrayList eventLoopGroupFutures = new ArrayList<>(3); - eventLoopGroupFutures.add(this.clientWorkerGroup.shutdownGracefully()); - eventLoopGroupFutures.add(this.serverBossGroup.shutdownGracefully()); - eventLoopGroupFutures.add(this.serverWorkerGroup.shutdownGracefully()); - - clientChannelGroupFuture.awaitUninterruptibly(); - serverChannelGroupFuture.awaitUninterruptibly(); - - try { - acceptorFuture.sync(); - } catch (final Exception ex) { - LOG.log(Level.SEVERE, "Error closing the acceptor channel for " + this.localAddress, ex); - } - - for (final Future eventLoopGroupFuture : eventLoopGroupFutures) { - eventLoopGroupFuture.awaitUninterruptibly(); - } - - LOG.log(Level.FINE, "Closing netty transport socket address: {0} done", this.localAddress); - } - - /** - * Returns a link for the remote address if cached; otherwise opens, caches and returns. - * When it opens a link for the remote address, only one attempt for the address is made at a given time - * - * @param remoteAddr the remote socket address - * @param encoder the encoder - * @param listener the link listener - * @return a link associated with the address - */ - @Override - public Link open(final SocketAddress remoteAddr, final Encoder encoder, - final LinkListener listener) throws IOException { - - Link link = null; - - for (int i = 0; i <= this.numberOfTries; ++i) { - LinkReference linkRef = this.addrToLinkRefMap.get(remoteAddr); - - if (linkRef != null) { - link = (Link) linkRef.getLink(); - if (LOG.isLoggable(Level.FINE)) { - LOG.log(Level.FINE, "Link {0} for {1} found", new Object[]{link, remoteAddr}); - } - if (link != null) { - return link; - } - } - - if (i == this.numberOfTries) { - // Connection failure - throw new ConnectException("Connection to " + remoteAddr + " refused"); - } - - LOG.log(Level.FINE, "No cached link for {0} thread {1}", - new Object[]{remoteAddr, Thread.currentThread()}); - - // no linkRef - final LinkReference newLinkRef = new LinkReference(); - final LinkReference prior = this.addrToLinkRefMap.putIfAbsent(remoteAddr, newLinkRef); - final AtomicInteger flag = prior != null ? - prior.getConnectInProgress() : newLinkRef.getConnectInProgress(); - - synchronized (flag) { - if (!flag.compareAndSet(0, 1)) { - while (flag.get() == 1) { - try { - flag.wait(); - } catch (final InterruptedException ex) { - LOG.log(Level.WARNING, "Wait interrupted", ex); - } - } - } - } - - linkRef = this.addrToLinkRefMap.get(remoteAddr); - link = (Link) linkRef.getLink(); - - if (link != null) { - return link; - } - - ChannelFuture connectFuture = null; - try { - connectFuture = this.clientBootstrap.connect(remoteAddr); - connectFuture.syncUninterruptibly(); - - link = new NettyHttpLink<>(connectFuture.channel(), encoder, listener, this.uri); - linkRef.setLink(link); - - synchronized (flag) { - flag.compareAndSet(1, 2); - flag.notifyAll(); - } - break; - } catch (final Exception e) { - if (e.getClass().getSimpleName().compareTo("ConnectException") == 0) { - LOG.log(Level.WARNING, "Connection refused. Retry {0} of {1}", - new Object[]{i + 1, this.numberOfTries}); - synchronized (flag) { - flag.compareAndSet(1, 0); - flag.notifyAll(); - } - - if (i < this.numberOfTries) { - try { - Thread.sleep(retryTimeout); - } catch (final InterruptedException interrupt) { - LOG.log(Level.WARNING, "Thread {0} interrupted while sleeping", Thread.currentThread()); - } - } - } else { - throw e; - } - } - } - - return link; - } - - /** - * Returns a link for the remote address if already cached; otherwise, returns null. - * - * @param remoteAddr the remote address - * @return a link if already cached; otherwise, null - */ - public Link get(final SocketAddress remoteAddr) { - final LinkReference linkRef = this.addrToLinkRefMap.get(remoteAddr); - return linkRef != null ? (Link) linkRef.getLink() : null; - } - - /** - * Gets a server local socket address of this transport. - * - * @return a server local socket address - */ - @Override - public SocketAddress getLocalAddress() { - return this.localAddress; - } - - /** - * Gets a server listening port of this transport. - * - * @return a listening port number - */ - @Override - public int getListeningPort() { - return this.serverPort; - } - - /** - * Registers the exception event handler. - * - * @param handler the exception event handler - */ - @Override - public void registerErrorHandler(final EventHandler handler) { - this.clientEventListener.registerErrorHandler(handler); - this.serverEventListener.registerErrorHandler(handler); - } -} diff --git a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/http/NettyHttpServerEventListener.java b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/http/NettyHttpServerEventListener.java deleted file mode 100644 index 54bfb19cd4..0000000000 --- a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/http/NettyHttpServerEventListener.java +++ /dev/null @@ -1,159 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.reef.wake.remote.transport.netty.http; - -import io.netty.buffer.ByteBuf; -import io.netty.channel.Channel; -import io.netty.channel.ChannelHandlerContext; -import io.netty.handler.codec.DecoderResult; -import io.netty.handler.codec.http.*; -import io.netty.util.CharsetUtil; -import org.apache.reef.wake.EStage; -import org.apache.reef.wake.remote.impl.ByteCodec; -import org.apache.reef.wake.remote.impl.TransportEvent; -import org.apache.reef.wake.remote.transport.netty.AbstractNettyEventListener; -import org.apache.reef.wake.remote.transport.netty.ByteEncoder; -import org.apache.reef.wake.remote.transport.netty.LinkReference; -import org.apache.reef.wake.remote.transport.netty.LoggingLinkListener; - -import java.net.SocketAddress; -import java.net.URI; -import java.util.Map; -import java.util.concurrent.ConcurrentMap; -import java.util.logging.Level; - -/** - * A Netty event listener for server side. - */ -final class NettyHttpServerEventListener extends AbstractNettyEventListener { - - private HttpRequest httpRequest; - private final StringBuilder buf = new StringBuilder(); - private final URI uri; - - NettyHttpServerEventListener( - final ConcurrentMap addrToLinkRefMap, - final EStage stage, - final URI uri) { - super(addrToLinkRefMap, stage); - this.uri = uri; - } - - - @Override - public void channelActive(final ChannelHandlerContext ctx) { - final Channel channel = ctx.channel(); - - if (LOG.isLoggable(Level.FINEST)) { - LOG.log(Level.FINEST, "Channel active. key: {0}", channel.remoteAddress()); - } - - this.addrToLinkRefMap.putIfAbsent( - channel.remoteAddress(), new LinkReference(new NettyHttpLink<>( - channel, new ByteCodec(), new LoggingLinkListener(), uri))); - - LOG.log(Level.FINER, "Add connected channel ref: {0}", this.addrToLinkRefMap.get(channel.remoteAddress())); - - } - - @Override - public void channelRead(final ChannelHandlerContext ctx, final Object msg) { - final HttpHeaders headers; - final HttpRequest request; - if(msg instanceof HttpRequest) { - LOG.log(Level.FINEST, "HttpRequest received"); - request = (HttpRequest) msg; - this.httpRequest = request; - headers = request.headers(); - if (!headers.isEmpty()) { - for (Map.Entry h: headers) { - CharSequence key = h.getKey(); - CharSequence value = h.getValue(); - buf.append("HEADER: ").append(key).append(" = ").append(value).append("\r\n"); - } - buf.append("\r\n"); - } - appendDecoderResult(buf, request); - } else if (msg instanceof HttpContent) { - LOG.log(Level.FINEST, "HttpContent received"); - HttpContent httpContent = (HttpContent) msg; - ByteBuf content = httpContent.content(); - if (content.isReadable()) { - buf.append("CONTENT: "); - buf.append(content.toString(CharsetUtil.UTF_8)); - buf.append("\r\n"); - appendDecoderResult(buf, this.httpRequest); - } - - if (msg instanceof LastHttpContent) { - buf.append("END OF CONTENT\r\n"); - LastHttpContent trailer = (LastHttpContent) msg; - if (!trailer.trailingHeaders().isEmpty()) { - buf.append("\r\n"); - for (CharSequence name: trailer.trailingHeaders().names()) { - for (CharSequence value: trailer.trailingHeaders().getAll(name)) { - buf.append("TRAILING HEADER: "); - buf.append(name).append(" = ").append(value).append("\r\n"); - } - } - buf.append("\r\n"); - } - - final Channel channel = ctx.channel(); - byte[] message = new byte[content.readableBytes()]; - content.readBytes(message); - if (LOG.isLoggable(Level.FINEST)) { - LOG.log(Level.FINEST, "MessageEvent: local: {0} remote: {1} :: {2}", new Object[]{ - channel.localAddress(), channel.remoteAddress(), content}); - } - - if (message.length > 0) { - // send to the dispatch stage - this.stage.onNext(this.getTransportEvent(message, channel)); - } - } - } else { - LOG.log(Level.SEVERE, "Unknown type of message received: {0}", msg); - } - LOG.log(Level.FINEST, buf.toString()); - } - - private static void appendDecoderResult(final StringBuilder buf, final HttpObject o) { - DecoderResult result = o.getDecoderResult(); - if (result.isSuccess()) { - return; - } - - buf.append(".. WITH DECODER FAILURE: "); - buf.append(result.cause()); - buf.append("\r\n"); - } - - - - @Override - protected TransportEvent getTransportEvent(final byte[] message, final Channel channel) { - return new TransportEvent(message, new NettyHttpLink<>(channel, new ByteEncoder())); - } - - @Override - protected void exceptionCleanup(final ChannelHandlerContext ctx, final Throwable cause) { - // noop - } -} diff --git a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/http/package-info.java b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/http/package-info.java deleted file mode 100644 index d4589468f4..0000000000 --- a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/http/package-info.java +++ /dev/null @@ -1,22 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -/** - * Wake's remote transportation by HTTP. - */ -package org.apache.reef.wake.remote.transport.netty.http; From 31e5efaf1093ec35e9766659413a356c79126e6b Mon Sep 17 00:00:00 2001 From: Kim Doyoung Date: Thu, 17 Aug 2017 18:29:53 +0900 Subject: [PATCH 07/34] fixed errors regarding to review --- .../remote/transport/TransportFactory.java | 18 +++++++------- .../netty/MessagingTransportFactory.java | 4 ++-- .../netty/NettyChannelFutureListener.java | 2 +- .../netty/NettyChannelInitializer.java | 8 +++++++ .../netty/NettyHttpClientEventListener.java | 10 ++++---- .../netty/NettyHttpServerEventListener.java | 24 +++++++++---------- .../remote/transport/netty/NettyLink.java | 4 ++-- .../netty/NettyMessagingTransport.java | 5 +++- 8 files changed, 43 insertions(+), 32 deletions(-) diff --git a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/TransportFactory.java b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/TransportFactory.java index a2d7eb40be..5382f3dc30 100644 --- a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/TransportFactory.java +++ b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/TransportFactory.java @@ -40,10 +40,10 @@ public interface TransportFactory { * @param exHandler an exception handler * @return transport */ - Transport newInstance(int port, - EventHandler clientHandler, - EventHandler serverHandler, - EventHandler exHandler); + Transport newInstance(final int port, + final EventHandler clientHandler, + final EventHandler serverHandler, + final EventHandler exHandler); /** * Creates a transport. @@ -56,7 +56,8 @@ Transport newInstance(int port, * @param retryTimeout retry timeout * @return transport */ - Transport newInstance(final String hostAddress, int port, + Transport newInstance(final String hostAddress, + final int port, final EStage clientStage, final EStage serverStage, final int numberOfTries, @@ -74,7 +75,8 @@ Transport newInstance(final String hostAddress, int port, * @param protocol protocol to use * @return transport */ - Transport newInstance(final String hostAddress, int port, + Transport newInstance(final String hostAddress, + final int port, final EStage clientStage, final EStage serverStage, final int numberOfTries, @@ -94,7 +96,7 @@ Transport newInstance(final String hostAddress, int port, * @return transport */ Transport newInstance(final String hostAddress, - int port, + final int port, final EStage clientStage, final EStage serverStage, final int numberOfTries, @@ -115,7 +117,7 @@ Transport newInstance(final String hostAddress, * @return transport */ Transport newInstance(final String hostAddress, - int port, + final int port, final EStage clientStage, final EStage serverStage, final int numberOfTries, diff --git a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/MessagingTransportFactory.java b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/MessagingTransportFactory.java index 2e9646ec69..0251ae581d 100644 --- a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/MessagingTransportFactory.java +++ b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/MessagingTransportFactory.java @@ -93,7 +93,7 @@ public Transport newInstance(final String hostAddress, final int numberOfTries, final int retryTimeout) { try { - TcpPortProvider tcpPortProvider = Tang.Factory.getTang().newInjector().getInstance(TcpPortProvider.class); + final TcpPortProvider tcpPortProvider = Tang.Factory.getTang().newInjector().getInstance(TcpPortProvider.class); return newInstance(hostAddress, port, clientStage, serverStage, numberOfTries, retryTimeout, tcpPortProvider); } catch (final InjectionException e) { @@ -120,7 +120,7 @@ public Transport newInstance(final String hostAddress, final int retryTimeout, final int protocol) { try { - TcpPortProvider tcpPortProvider = Tang.Factory.getTang().newInjector().getInstance(TcpPortProvider.class); + final TcpPortProvider tcpPortProvider = Tang.Factory.getTang().newInjector().getInstance(TcpPortProvider.class); return newInstance(hostAddress, port, clientStage, serverStage, numberOfTries, retryTimeout, tcpPortProvider); } catch (final InjectionException e) { diff --git a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyChannelFutureListener.java b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyChannelFutureListener.java index 3fefc5a395..6b7a450ce2 100644 --- a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyChannelFutureListener.java +++ b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyChannelFutureListener.java @@ -22,7 +22,7 @@ import io.netty.channel.ChannelFutureListener; import org.apache.reef.wake.remote.transport.LinkListener; -class NettyChannelFutureListener implements ChannelFutureListener { +public final class NettyChannelFutureListener implements ChannelFutureListener { private final T message; private LinkListener listener; diff --git a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyChannelInitializer.java b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyChannelInitializer.java index 80962a2046..d0ff959f03 100644 --- a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyChannelInitializer.java +++ b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyChannelInitializer.java @@ -41,7 +41,15 @@ class NettyChannelInitializer extends ChannelInitializer { */ public static final int MAXFRAMELENGTH = 10 * 1024 * 1024; private final NettyChannelHandlerFactory handlerFactory; + + /** + * sslContext contains ssl context of the machine. used only for HTTP. + */ private final SslContext sslContext; + + /** + * Type of channel whether it is netty or http client or http server. + */ private final int type; NettyChannelInitializer( diff --git a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyHttpClientEventListener.java b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyHttpClientEventListener.java index bb10883dde..2d5e4a4a0d 100644 --- a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyHttpClientEventListener.java +++ b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyHttpClientEventListener.java @@ -50,11 +50,10 @@ final class NettyHttpClientEventListener extends AbstractNettyEventListener { @Override public void channelRead(final ChannelHandlerContext ctx, final Object msg) { if (msg instanceof HttpResponse) { - System.out.println("CLIENT : IT IS HTTP RESPONSE"); + LOG.log(Level.FINEST, "HttpResponse Received: {0}", msg); } else if (msg instanceof HttpContent) { - System.out.println("CLIENT : It is HttpContent"); - HttpContent httpContent = (HttpContent) msg; - ByteBuf content = httpContent.content(); + final HttpContent httpContent = (HttpContent) msg; + final ByteBuf content = httpContent.content(); if (content.isReadable()) { buf.append("CONTENT: "); buf.append(content.toString(CharsetUtil.UTF_8)); @@ -63,7 +62,7 @@ public void channelRead(final ChannelHandlerContext ctx, final Object msg) { if (msg instanceof LastHttpContent) { final Channel channel = ctx.channel(); - byte[] message = new byte[content.readableBytes()]; + final byte[] message = new byte[content.readableBytes()]; content.readBytes(message); if (LOG.isLoggable(Level.FINEST)) { LOG.log(Level.FINEST, "MessageEvent: local: {0} remote: {1} :: {2}", new Object[]{ @@ -84,7 +83,6 @@ public void channelRead(final ChannelHandlerContext ctx, final Object msg) { @Override public void channelActive(final ChannelHandlerContext ctx) { // noop - LOG.log(Level.FINEST, "{0}", ctx); } @Override diff --git a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyHttpServerEventListener.java b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyHttpServerEventListener.java index 00e6ca638d..a085be2ce2 100644 --- a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyHttpServerEventListener.java +++ b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyHttpServerEventListener.java @@ -70,17 +70,17 @@ public void channelActive(final ChannelHandlerContext ctx) { @Override public void channelRead(final ChannelHandlerContext ctx, final Object msg) { - final HttpHeaders headers; - final HttpRequest request; if(msg instanceof HttpRequest) { LOG.log(Level.FINEST, "HttpRequest received"); - request = (HttpRequest) msg; + + final HttpRequest request = (HttpRequest) msg; + final HttpHeaders headers = request.headers(); this.httpRequest = request; - headers = request.headers(); + if (!headers.isEmpty()) { - for (Map.Entry h: headers) { - CharSequence key = h.getKey(); - CharSequence value = h.getValue(); + for (final Map.Entry h: headers) { + final CharSequence key = h.getKey(); + final CharSequence value = h.getValue(); buf.append("HEADER: ").append(key).append(" = ").append(value).append("\r\n"); } buf.append("\r\n"); @@ -88,8 +88,8 @@ public void channelRead(final ChannelHandlerContext ctx, final Object msg) { appendDecoderResult(buf, request); } else if (msg instanceof HttpContent) { LOG.log(Level.FINEST, "HttpContent received"); - HttpContent httpContent = (HttpContent) msg; - ByteBuf content = httpContent.content(); + final HttpContent httpContent = (HttpContent) msg; + final ByteBuf content = httpContent.content(); if (content.isReadable()) { buf.append("CONTENT: "); buf.append(content.toString(CharsetUtil.UTF_8)); @@ -99,7 +99,7 @@ public void channelRead(final ChannelHandlerContext ctx, final Object msg) { if (msg instanceof LastHttpContent) { buf.append("END OF CONTENT\r\n"); - LastHttpContent trailer = (LastHttpContent) msg; + final LastHttpContent trailer = (LastHttpContent) msg; if (!trailer.trailingHeaders().isEmpty()) { buf.append("\r\n"); for (CharSequence name: trailer.trailingHeaders().names()) { @@ -112,7 +112,7 @@ public void channelRead(final ChannelHandlerContext ctx, final Object msg) { } final Channel channel = ctx.channel(); - byte[] message = new byte[content.readableBytes()]; + final byte[] message = new byte[content.readableBytes()]; content.readBytes(message); if (LOG.isLoggable(Level.FINEST)) { LOG.log(Level.FINEST, "MessageEvent: local: {0} remote: {1} :: {2}", new Object[]{ @@ -131,7 +131,7 @@ public void channelRead(final ChannelHandlerContext ctx, final Object msg) { } private static void appendDecoderResult(final StringBuilder buf, final HttpObject o) { - DecoderResult result = o.getDecoderResult(); + final DecoderResult result = o.getDecoderResult(); if (result.isSuccess()) { return; } diff --git a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyLink.java b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyLink.java index a5d7b74785..ff01bd3496 100644 --- a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyLink.java +++ b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyLink.java @@ -99,12 +99,12 @@ public void write(final T message) { if (this.uri != null) { try { - FullHttpRequest request = new DefaultFullHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.POST, uri.getRawPath()); + final FullHttpRequest request = new DefaultFullHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.POST, uri.getRawPath()); request.headers().set(HttpHeaders.Names.HOST, uri.getHost()); request.headers().set(HttpHeaders.Names.CONNECTION, HttpHeaders.Values.KEEP_ALIVE); request.headers().set(HttpHeaders.Names.ACCEPT_ENCODING, HttpHeaders.Values.GZIP); request.headers().set(HttpHeaders.Names.CONTENT_TYPE, "application/wake-transport"); - ByteBuf buf = Unpooled.copiedBuffer(encoder.encode(message)); + final ByteBuf buf = Unpooled.copiedBuffer(encoder.encode(message)); request.headers().set(HttpHeaders.Names.CONTENT_LENGTH, buf.readableBytes()); request.content().clear().writeBytes(buf); final ChannelFuture future = channel.writeAndFlush(request); diff --git a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyMessagingTransport.java b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyMessagingTransport.java index 3bc355eee1..6f4c247459 100644 --- a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyMessagingTransport.java +++ b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyMessagingTransport.java @@ -104,7 +104,7 @@ public final class NettyMessagingTransport implements Transport { private final AbstractNettyEventListener clientEventListener; private final AbstractNettyEventListener serverEventListener; - private final boolean ssl = System.getProperty("ssl") != null; + private final boolean ssl; private final URI uri; private final int numberOfTries; @@ -143,6 +143,9 @@ private NettyMessagingTransport( final SslContext sslContextClient; final SslContext sslContextServer; + + ssl = System.getProperty("ssl") != null; + if(protocolType == PROTOCOL_HTTP) { if (ssl) { try { From bc42c3dc6462e8f5a529d364d437511bc0f4e054 Mon Sep 17 00:00:00 2001 From: Kim Doyoung Date: Thu, 17 Aug 2017 18:29:53 +0900 Subject: [PATCH 08/34] fixed errors regarding to review --- .../remote/transport/TransportFactory.java | 18 +++++++------- .../netty/MessagingTransportFactory.java | 4 ++-- .../netty/NettyChannelFutureListener.java | 2 +- .../netty/NettyChannelInitializer.java | 8 +++++++ .../netty/NettyHttpClientEventListener.java | 10 ++++---- .../netty/NettyHttpServerEventListener.java | 24 +++++++++---------- .../remote/transport/netty/NettyLink.java | 6 ++--- .../netty/NettyMessagingTransport.java | 5 +++- 8 files changed, 44 insertions(+), 33 deletions(-) diff --git a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/TransportFactory.java b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/TransportFactory.java index a2d7eb40be..5382f3dc30 100644 --- a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/TransportFactory.java +++ b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/TransportFactory.java @@ -40,10 +40,10 @@ public interface TransportFactory { * @param exHandler an exception handler * @return transport */ - Transport newInstance(int port, - EventHandler clientHandler, - EventHandler serverHandler, - EventHandler exHandler); + Transport newInstance(final int port, + final EventHandler clientHandler, + final EventHandler serverHandler, + final EventHandler exHandler); /** * Creates a transport. @@ -56,7 +56,8 @@ Transport newInstance(int port, * @param retryTimeout retry timeout * @return transport */ - Transport newInstance(final String hostAddress, int port, + Transport newInstance(final String hostAddress, + final int port, final EStage clientStage, final EStage serverStage, final int numberOfTries, @@ -74,7 +75,8 @@ Transport newInstance(final String hostAddress, int port, * @param protocol protocol to use * @return transport */ - Transport newInstance(final String hostAddress, int port, + Transport newInstance(final String hostAddress, + final int port, final EStage clientStage, final EStage serverStage, final int numberOfTries, @@ -94,7 +96,7 @@ Transport newInstance(final String hostAddress, int port, * @return transport */ Transport newInstance(final String hostAddress, - int port, + final int port, final EStage clientStage, final EStage serverStage, final int numberOfTries, @@ -115,7 +117,7 @@ Transport newInstance(final String hostAddress, * @return transport */ Transport newInstance(final String hostAddress, - int port, + final int port, final EStage clientStage, final EStage serverStage, final int numberOfTries, diff --git a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/MessagingTransportFactory.java b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/MessagingTransportFactory.java index 2e9646ec69..0251ae581d 100644 --- a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/MessagingTransportFactory.java +++ b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/MessagingTransportFactory.java @@ -93,7 +93,7 @@ public Transport newInstance(final String hostAddress, final int numberOfTries, final int retryTimeout) { try { - TcpPortProvider tcpPortProvider = Tang.Factory.getTang().newInjector().getInstance(TcpPortProvider.class); + final TcpPortProvider tcpPortProvider = Tang.Factory.getTang().newInjector().getInstance(TcpPortProvider.class); return newInstance(hostAddress, port, clientStage, serverStage, numberOfTries, retryTimeout, tcpPortProvider); } catch (final InjectionException e) { @@ -120,7 +120,7 @@ public Transport newInstance(final String hostAddress, final int retryTimeout, final int protocol) { try { - TcpPortProvider tcpPortProvider = Tang.Factory.getTang().newInjector().getInstance(TcpPortProvider.class); + final TcpPortProvider tcpPortProvider = Tang.Factory.getTang().newInjector().getInstance(TcpPortProvider.class); return newInstance(hostAddress, port, clientStage, serverStage, numberOfTries, retryTimeout, tcpPortProvider); } catch (final InjectionException e) { diff --git a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyChannelFutureListener.java b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyChannelFutureListener.java index 3fefc5a395..6b7a450ce2 100644 --- a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyChannelFutureListener.java +++ b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyChannelFutureListener.java @@ -22,7 +22,7 @@ import io.netty.channel.ChannelFutureListener; import org.apache.reef.wake.remote.transport.LinkListener; -class NettyChannelFutureListener implements ChannelFutureListener { +public final class NettyChannelFutureListener implements ChannelFutureListener { private final T message; private LinkListener listener; diff --git a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyChannelInitializer.java b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyChannelInitializer.java index 80962a2046..d0ff959f03 100644 --- a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyChannelInitializer.java +++ b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyChannelInitializer.java @@ -41,7 +41,15 @@ class NettyChannelInitializer extends ChannelInitializer { */ public static final int MAXFRAMELENGTH = 10 * 1024 * 1024; private final NettyChannelHandlerFactory handlerFactory; + + /** + * sslContext contains ssl context of the machine. used only for HTTP. + */ private final SslContext sslContext; + + /** + * Type of channel whether it is netty or http client or http server. + */ private final int type; NettyChannelInitializer( diff --git a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyHttpClientEventListener.java b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyHttpClientEventListener.java index bb10883dde..2d5e4a4a0d 100644 --- a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyHttpClientEventListener.java +++ b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyHttpClientEventListener.java @@ -50,11 +50,10 @@ final class NettyHttpClientEventListener extends AbstractNettyEventListener { @Override public void channelRead(final ChannelHandlerContext ctx, final Object msg) { if (msg instanceof HttpResponse) { - System.out.println("CLIENT : IT IS HTTP RESPONSE"); + LOG.log(Level.FINEST, "HttpResponse Received: {0}", msg); } else if (msg instanceof HttpContent) { - System.out.println("CLIENT : It is HttpContent"); - HttpContent httpContent = (HttpContent) msg; - ByteBuf content = httpContent.content(); + final HttpContent httpContent = (HttpContent) msg; + final ByteBuf content = httpContent.content(); if (content.isReadable()) { buf.append("CONTENT: "); buf.append(content.toString(CharsetUtil.UTF_8)); @@ -63,7 +62,7 @@ public void channelRead(final ChannelHandlerContext ctx, final Object msg) { if (msg instanceof LastHttpContent) { final Channel channel = ctx.channel(); - byte[] message = new byte[content.readableBytes()]; + final byte[] message = new byte[content.readableBytes()]; content.readBytes(message); if (LOG.isLoggable(Level.FINEST)) { LOG.log(Level.FINEST, "MessageEvent: local: {0} remote: {1} :: {2}", new Object[]{ @@ -84,7 +83,6 @@ public void channelRead(final ChannelHandlerContext ctx, final Object msg) { @Override public void channelActive(final ChannelHandlerContext ctx) { // noop - LOG.log(Level.FINEST, "{0}", ctx); } @Override diff --git a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyHttpServerEventListener.java b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyHttpServerEventListener.java index 00e6ca638d..a085be2ce2 100644 --- a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyHttpServerEventListener.java +++ b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyHttpServerEventListener.java @@ -70,17 +70,17 @@ public void channelActive(final ChannelHandlerContext ctx) { @Override public void channelRead(final ChannelHandlerContext ctx, final Object msg) { - final HttpHeaders headers; - final HttpRequest request; if(msg instanceof HttpRequest) { LOG.log(Level.FINEST, "HttpRequest received"); - request = (HttpRequest) msg; + + final HttpRequest request = (HttpRequest) msg; + final HttpHeaders headers = request.headers(); this.httpRequest = request; - headers = request.headers(); + if (!headers.isEmpty()) { - for (Map.Entry h: headers) { - CharSequence key = h.getKey(); - CharSequence value = h.getValue(); + for (final Map.Entry h: headers) { + final CharSequence key = h.getKey(); + final CharSequence value = h.getValue(); buf.append("HEADER: ").append(key).append(" = ").append(value).append("\r\n"); } buf.append("\r\n"); @@ -88,8 +88,8 @@ public void channelRead(final ChannelHandlerContext ctx, final Object msg) { appendDecoderResult(buf, request); } else if (msg instanceof HttpContent) { LOG.log(Level.FINEST, "HttpContent received"); - HttpContent httpContent = (HttpContent) msg; - ByteBuf content = httpContent.content(); + final HttpContent httpContent = (HttpContent) msg; + final ByteBuf content = httpContent.content(); if (content.isReadable()) { buf.append("CONTENT: "); buf.append(content.toString(CharsetUtil.UTF_8)); @@ -99,7 +99,7 @@ public void channelRead(final ChannelHandlerContext ctx, final Object msg) { if (msg instanceof LastHttpContent) { buf.append("END OF CONTENT\r\n"); - LastHttpContent trailer = (LastHttpContent) msg; + final LastHttpContent trailer = (LastHttpContent) msg; if (!trailer.trailingHeaders().isEmpty()) { buf.append("\r\n"); for (CharSequence name: trailer.trailingHeaders().names()) { @@ -112,7 +112,7 @@ public void channelRead(final ChannelHandlerContext ctx, final Object msg) { } final Channel channel = ctx.channel(); - byte[] message = new byte[content.readableBytes()]; + final byte[] message = new byte[content.readableBytes()]; content.readBytes(message); if (LOG.isLoggable(Level.FINEST)) { LOG.log(Level.FINEST, "MessageEvent: local: {0} remote: {1} :: {2}", new Object[]{ @@ -131,7 +131,7 @@ public void channelRead(final ChannelHandlerContext ctx, final Object msg) { } private static void appendDecoderResult(final StringBuilder buf, final HttpObject o) { - DecoderResult result = o.getDecoderResult(); + final DecoderResult result = o.getDecoderResult(); if (result.isSuccess()) { return; } diff --git a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyLink.java b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyLink.java index a5d7b74785..5b55fe4341 100644 --- a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyLink.java +++ b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyLink.java @@ -99,12 +99,12 @@ public void write(final T message) { if (this.uri != null) { try { - FullHttpRequest request = new DefaultFullHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.POST, uri.getRawPath()); + final FullHttpRequest request = new DefaultFullHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.POST, uri.getRawPath()); request.headers().set(HttpHeaders.Names.HOST, uri.getHost()); request.headers().set(HttpHeaders.Names.CONNECTION, HttpHeaders.Values.KEEP_ALIVE); request.headers().set(HttpHeaders.Names.ACCEPT_ENCODING, HttpHeaders.Values.GZIP); request.headers().set(HttpHeaders.Names.CONTENT_TYPE, "application/wake-transport"); - ByteBuf buf = Unpooled.copiedBuffer(encoder.encode(message)); + final ByteBuf buf = Unpooled.copiedBuffer(encoder.encode(message)); request.headers().set(HttpHeaders.Names.CONTENT_LENGTH, buf.readableBytes()); request.content().clear().writeBytes(buf); final ChannelFuture future = channel.writeAndFlush(request); @@ -112,7 +112,7 @@ public void write(final T message) { if (listener != null) { future.addListener(new NettyChannelFutureListener<>(message, listener)); } - } catch (InterruptedException ex) { + } catch (final InterruptedException ex) { LOG.log(Level.SEVERE, "Cannot send request to " + uri.getHost(), ex); } } else { diff --git a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyMessagingTransport.java b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyMessagingTransport.java index 3bc355eee1..6f4c247459 100644 --- a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyMessagingTransport.java +++ b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyMessagingTransport.java @@ -104,7 +104,7 @@ public final class NettyMessagingTransport implements Transport { private final AbstractNettyEventListener clientEventListener; private final AbstractNettyEventListener serverEventListener; - private final boolean ssl = System.getProperty("ssl") != null; + private final boolean ssl; private final URI uri; private final int numberOfTries; @@ -143,6 +143,9 @@ private NettyMessagingTransport( final SslContext sslContextClient; final SslContext sslContextServer; + + ssl = System.getProperty("ssl") != null; + if(protocolType == PROTOCOL_HTTP) { if (ssl) { try { From 194360909ca9f0696adda0cb32e0f66a736ff36d Mon Sep 17 00:00:00 2001 From: Kim Doyoung Date: Mon, 21 Aug 2017 17:56:18 +0900 Subject: [PATCH 09/34] Fixed errors and typos --- .../reef/wake/remote/RemoteConfiguration.java | 8 ++--- .../remote/transport/TransportFactory.java | 22 ++++++------- .../netty/MessagingTransportFactory.java | 13 ++++---- .../netty/NettyChannelInitializer.java | 17 +++++----- .../netty/NettyHttpClientEventListener.java | 12 +++---- .../netty/NettyHttpServerEventListener.java | 12 +++---- .../remote/transport/netty/NettyLink.java | 15 ++++----- .../netty/NettyMessagingTransport.java | 31 +++++++++---------- .../wake/test/remote/TransportHttpTest.java | 6 ++-- 9 files changed, 67 insertions(+), 69 deletions(-) diff --git a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/RemoteConfiguration.java b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/RemoteConfiguration.java index 9343205828..8cc1291a4d 100644 --- a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/RemoteConfiguration.java +++ b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/RemoteConfiguration.java @@ -51,8 +51,8 @@ public final class RemoteConfiguration { /** * Unique protocol numbers for choosing protocols. */ - public static final int PROTOCOL_NETTY = 100; - public static final int PROTOCOL_HTTP = 101; + public static final String PROTOCOL_NETTY = "__PROTOCOL_NETTY__"; + public static final String PROTOCOL_HTTP = "__PROTOCOL_HTTP__"; private RemoteConfiguration() { // empty @@ -143,8 +143,8 @@ public static final class RemoteServerStage implements Name { + @NamedParameter(doc = "Option for use http.", default_value = PROTOCOL_NETTY) + public static final class Protocol implements Name { // Intentionally empty } } diff --git a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/TransportFactory.java b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/TransportFactory.java index 5382f3dc30..18576dc053 100644 --- a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/TransportFactory.java +++ b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/TransportFactory.java @@ -40,10 +40,10 @@ public interface TransportFactory { * @param exHandler an exception handler * @return transport */ - Transport newInstance(final int port, - final EventHandler clientHandler, - final EventHandler serverHandler, - final EventHandler exHandler); + Transport newInstance(int port, + EventHandler clientHandler, + EventHandler serverHandler, + EventHandler exHandler); /** * Creates a transport. @@ -56,8 +56,7 @@ Transport newInstance(final int port, * @param retryTimeout retry timeout * @return transport */ - Transport newInstance(final String hostAddress, - final int port, + Transport newInstance(final String hostAddress, int port, final EStage clientStage, final EStage serverStage, final int numberOfTries, @@ -75,13 +74,12 @@ Transport newInstance(final String hostAddress, * @param protocol protocol to use * @return transport */ - Transport newInstance(final String hostAddress, - final int port, + Transport newInstance(final String hostAddress, int port, final EStage clientStage, final EStage serverStage, final int numberOfTries, final int retryTimeout, - final int protocol); + final String protocol); /** * Creates a transport. @@ -96,7 +94,7 @@ Transport newInstance(final String hostAddress, * @return transport */ Transport newInstance(final String hostAddress, - final int port, + int port, final EStage clientStage, final EStage serverStage, final int numberOfTries, @@ -117,13 +115,13 @@ Transport newInstance(final String hostAddress, * @return transport */ Transport newInstance(final String hostAddress, - final int port, + int port, final EStage clientStage, final EStage serverStage, final int numberOfTries, final int retryTimeout, final TcpPortProvider tcpPortProvider, - final int protocol); + final String protocol); } diff --git a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/MessagingTransportFactory.java b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/MessagingTransportFactory.java index 0251ae581d..bd745dfa31 100644 --- a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/MessagingTransportFactory.java +++ b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/MessagingTransportFactory.java @@ -39,6 +39,7 @@ public final class MessagingTransportFactory implements TransportFactory { private final String localAddress; + private static final Tang TANG = Tang.Factory.getTang(); @Inject private MessagingTransportFactory(final LocalAddressProvider localAddressProvider) { @@ -59,7 +60,7 @@ public Transport newInstance(final int port, final EventHandler serverHandler, final EventHandler exHandler) { - final Injector injector = Tang.Factory.getTang().newInjector(); + final Injector injector = TANG.newInjector(); injector.bindVolatileParameter(RemoteConfiguration.HostAddress.class, this.localAddress); injector.bindVolatileParameter(RemoteConfiguration.Port.class, port); injector.bindVolatileParameter(RemoteConfiguration.RemoteClientStage.class, new SyncStage<>(clientHandler)); @@ -93,7 +94,7 @@ public Transport newInstance(final String hostAddress, final int numberOfTries, final int retryTimeout) { try { - final TcpPortProvider tcpPortProvider = Tang.Factory.getTang().newInjector().getInstance(TcpPortProvider.class); + final TcpPortProvider tcpPortProvider = TANG.newInjector().getInstance(TcpPortProvider.class); return newInstance(hostAddress, port, clientStage, serverStage, numberOfTries, retryTimeout, tcpPortProvider); } catch (final InjectionException e) { @@ -118,9 +119,9 @@ public Transport newInstance(final String hostAddress, final EStage serverStage, final int numberOfTries, final int retryTimeout, - final int protocol) { + final String protocol) { try { - final TcpPortProvider tcpPortProvider = Tang.Factory.getTang().newInjector().getInstance(TcpPortProvider.class); + final TcpPortProvider tcpPortProvider = TANG.newInjector().getInstance(TcpPortProvider.class); return newInstance(hostAddress, port, clientStage, serverStage, numberOfTries, retryTimeout, tcpPortProvider); } catch (final InjectionException e) { @@ -172,9 +173,9 @@ public Transport newInstance(final String hostAddress, final int numberOfTries, final int retryTimeout, final TcpPortProvider tcpPortProvider, - final int protocol) { + final String protocol) { - final Injector injector = Tang.Factory.getTang().newInjector(); + final Injector injector = TANG.newInjector(); injector.bindVolatileParameter(RemoteConfiguration.HostAddress.class, hostAddress); injector.bindVolatileParameter(RemoteConfiguration.Port.class, port); injector.bindVolatileParameter(RemoteConfiguration.RemoteClientStage.class, clientStage); diff --git a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyChannelInitializer.java b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyChannelInitializer.java index d0ff959f03..7a72e4f015 100644 --- a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyChannelInitializer.java +++ b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyChannelInitializer.java @@ -32,9 +32,10 @@ */ class NettyChannelInitializer extends ChannelInitializer { - private static final int HANDLER_NETTY = 100; - private static final int HANDLER_HTTP_SERVER = 101; - private static final int HANDLER_HTTP_CLIENT = 102; + /* Types for initiating channel */ + public enum ChannelType { + NETTY, HTTP_SERVER, HTTP_CLIENT + } /** * the buffer size of the frame decoder. @@ -50,12 +51,12 @@ class NettyChannelInitializer extends ChannelInitializer { /** * Type of channel whether it is netty or http client or http server. */ - private final int type; + private final ChannelType type; NettyChannelInitializer( final NettyChannelHandlerFactory handlerFactory, final SslContext sslContext, - final int type) { + final ChannelType type) { this.handlerFactory = handlerFactory; this.sslContext = sslContext; this.type = type; @@ -64,7 +65,7 @@ class NettyChannelInitializer extends ChannelInitializer { @Override protected void initChannel(final SocketChannel ch) throws Exception { switch (this.type) { - case HANDLER_NETTY: + case NETTY: ch.pipeline() .addLast("frameDecoder", new LengthFieldBasedFrameDecoder(MAXFRAMELENGTH, 0, 4, 0, 4)) .addLast("bytesDecoder", new ByteArrayDecoder()) @@ -73,7 +74,7 @@ protected void initChannel(final SocketChannel ch) throws Exception { .addLast("chunker", new ChunkedReadWriteHandler()) .addLast("handler", handlerFactory.createChannelInboundHandler()); break; - case HANDLER_HTTP_SERVER: + case HTTP_SERVER: if (sslContext != null) { ch.pipeline().addLast(sslContext.newHandler(ch.alloc())); } @@ -83,7 +84,7 @@ protected void initChannel(final SocketChannel ch) throws Exception { .addLast("responseEncoder", new HttpResponseEncoder()) .addLast("handler", handlerFactory.createChannelInboundHandler()); break; - case HANDLER_HTTP_CLIENT: + case HTTP_CLIENT: if (sslContext != null) { ch.pipeline().addLast(sslContext.newHandler(ch.alloc())); } diff --git a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyHttpClientEventListener.java b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyHttpClientEventListener.java index 2d5e4a4a0d..5daaf2a267 100644 --- a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyHttpClientEventListener.java +++ b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyHttpClientEventListener.java @@ -39,7 +39,7 @@ final class NettyHttpClientEventListener extends AbstractNettyEventListener { private static final Logger LOG = Logger.getLogger(NettyHttpClientEventListener.class.getName()); - private StringBuilder buf = new StringBuilder(); + private final StringBuilder buf = new StringBuilder(); NettyHttpClientEventListener( final ConcurrentMap addrToLinkRefMap, @@ -50,14 +50,14 @@ final class NettyHttpClientEventListener extends AbstractNettyEventListener { @Override public void channelRead(final ChannelHandlerContext ctx, final Object msg) { if (msg instanceof HttpResponse) { - LOG.log(Level.FINEST, "HttpResponse Received: {0}", msg); + if(LOG.isLoggable(Level.FINEST)) { + LOG.log(Level.FINEST, "HttpResponse Received: {0}", msg); + } } else if (msg instanceof HttpContent) { final HttpContent httpContent = (HttpContent) msg; final ByteBuf content = httpContent.content(); - if (content.isReadable()) { - buf.append("CONTENT: "); - buf.append(content.toString(CharsetUtil.UTF_8)); - buf.append("\r\n"); + if (content.isReadable() && LOG.isLoggable(Level.FINEST)) { + buf.append("CONTENT: ").append(content.toString(CharsetUtil.UTF_8)).append("\r\n"); } if (msg instanceof LastHttpContent) { diff --git a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyHttpServerEventListener.java b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyHttpServerEventListener.java index a085be2ce2..377f15858e 100644 --- a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyHttpServerEventListener.java +++ b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyHttpServerEventListener.java @@ -127,18 +127,16 @@ public void channelRead(final ChannelHandlerContext ctx, final Object msg) { } else { LOG.log(Level.SEVERE, "Unknown type of message received: {0}", msg); } - LOG.log(Level.FINEST, buf.toString()); + if(LOG.isLoggable(Level.FINEST)) { + LOG.log(Level.FINEST, "Message received {0}", buf); + } } private static void appendDecoderResult(final StringBuilder buf, final HttpObject o) { final DecoderResult result = o.getDecoderResult(); - if (result.isSuccess()) { - return; + if (!result.isSuccess()) { + buf.append(".. WITH DECODER FAILURE: ").append(result.cause()).append("\r\n"); } - - buf.append(".. WITH DECODER FAILURE: "); - buf.append(result.cause()); - buf.append("\r\n"); } diff --git a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyLink.java b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyLink.java index 5b55fe4341..10524f5353 100644 --- a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyLink.java +++ b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyLink.java @@ -100,16 +100,17 @@ public void write(final T message) { if (this.uri != null) { try { final FullHttpRequest request = new DefaultFullHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.POST, uri.getRawPath()); - request.headers().set(HttpHeaders.Names.HOST, uri.getHost()); - request.headers().set(HttpHeaders.Names.CONNECTION, HttpHeaders.Values.KEEP_ALIVE); - request.headers().set(HttpHeaders.Names.ACCEPT_ENCODING, HttpHeaders.Values.GZIP); - request.headers().set(HttpHeaders.Names.CONTENT_TYPE, "application/wake-transport"); final ByteBuf buf = Unpooled.copiedBuffer(encoder.encode(message)); - request.headers().set(HttpHeaders.Names.CONTENT_LENGTH, buf.readableBytes()); + request.headers() + .set(HttpHeaders.Names.HOST, uri.getHost()) + .set(HttpHeaders.Names.CONNECTION, HttpHeaders.Values.KEEP_ALIVE) + .set(HttpHeaders.Names.ACCEPT_ENCODING, HttpHeaders.Values.GZIP) + .set(HttpHeaders.Names.CONTENT_TYPE, "application/wake-transport") + .set(HttpHeaders.Names.CONTENT_LENGTH, buf.readableBytes()); request.content().clear().writeBytes(buf); final ChannelFuture future = channel.writeAndFlush(request); future.sync(); - if (listener != null) { + if (listener != null) { future.addListener(new NettyChannelFutureListener<>(message, listener)); } } catch (final InterruptedException ex) { @@ -117,7 +118,7 @@ public void write(final T message) { } } else { final ChannelFuture future = channel.writeAndFlush(Unpooled.wrappedBuffer(encoder.encode(message))); - if (listener != null) { + if (listener != null) { future.addListener(new NettyChannelFutureListener<>(message, listener)); } } diff --git a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyMessagingTransport.java b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyMessagingTransport.java index 6f4c247459..e0beace776 100644 --- a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyMessagingTransport.java +++ b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyMessagingTransport.java @@ -62,6 +62,7 @@ import static org.apache.reef.wake.remote.RemoteConfiguration.PROTOCOL_HTTP; import static org.apache.reef.wake.remote.RemoteConfiguration.PROTOCOL_NETTY; +import static org.apache.reef.wake.remote.transport.netty.NettyChannelInitializer.ChannelType; /** * Messaging transport implementation with Netty and Http. @@ -81,10 +82,6 @@ public final class NettyMessagingTransport implements Transport { private static final int SERVER_WORKER_NUM_THREADS = 20; private static final int CLIENT_WORKER_NUM_THREADS = 10; - private static final int HANDLER_NETTY = 100; - private static final int HANDLER_HTTP_SERVER = 101; - private static final int HANDLER_HTTP_CLIENT = 102; - private final ConcurrentMap addrToLinkRefMap = new ConcurrentHashMap<>(); private final EventLoopGroup clientWorkerGroup; @@ -104,7 +101,7 @@ public final class NettyMessagingTransport implements Transport { private final AbstractNettyEventListener clientEventListener; private final AbstractNettyEventListener serverEventListener; - private final boolean ssl; + private final boolean supportsSSL; private final URI uri; private final int numberOfTries; @@ -132,7 +129,7 @@ private NettyMessagingTransport( @Parameter(RemoteConfiguration.RetryTimeout.class) final int retryTimeout, final TcpPortProvider tcpPortProvider, final LocalAddressProvider localAddressProvider, - @Parameter(RemoteConfiguration.Protocol.class) final int protocolType) { + @Parameter(RemoteConfiguration.Protocol.class) final String protocolType) { int p = port; if (p < 0) { @@ -144,19 +141,19 @@ private NettyMessagingTransport( final SslContext sslContextClient; final SslContext sslContextServer; - ssl = System.getProperty("ssl") != null; + supportsSSL = System.getProperty("ssl") != null; - if(protocolType == PROTOCOL_HTTP) { - if (ssl) { + if (protocolType.equals(PROTOCOL_HTTP)) { + if (supportsSSL) { try { - SelfSignedCertificate ssc = new SelfSignedCertificate(); + final SelfSignedCertificate ssc = new SelfSignedCertificate(); sslContextClient = SslContext.newServerContext(ssc.certificate(), ssc.privateKey()); sslContextServer = SslContext.newClientContext(); LOG.log(Level.FINE, "SSL context created"); - } catch (Exception ex) { + } catch (final Exception ex) { final RuntimeException transportException = - new TransportRuntimeException("Could create SSL Context", ex); - LOG.log(Level.SEVERE, "Cannot create SSL Context", ex); + new TransportRuntimeException("Could not create SSL Context", ex); + LOG.log(Level.SEVERE, "Could not create SSL Context", ex); throw transportException; } } else { @@ -164,7 +161,7 @@ private NettyMessagingTransport( sslContextClient = null; sslContextServer = null; } - this.uri = URI.create(ssl ? "https://" : "http://" + hostAddress); + this.uri = URI.create((supportsSSL ? "https://" : "http://") + hostAddress); } else { sslContextClient = null; sslContextServer = null; @@ -173,7 +170,7 @@ private NettyMessagingTransport( this.numberOfTries = numberOfTries; this.retryTimeout = retryTimeout; - if (protocolType == PROTOCOL_NETTY) { + if (protocolType.equals(PROTOCOL_NETTY)) { this.clientEventListener = new NettyClientEventListener(this.addrToLinkRefMap, clientStage); this.serverEventListener = new NettyServerEventListener(this.addrToLinkRefMap, serverStage); } else { @@ -194,7 +191,7 @@ private NettyMessagingTransport( .channel(NioSocketChannel.class) .handler(new NettyChannelInitializer(new NettyDefaultChannelHandlerFactory("client", this.clientChannelGroup, this.clientEventListener), sslContextClient, - protocolType == PROTOCOL_NETTY ? HANDLER_NETTY : HANDLER_HTTP_CLIENT)) + protocolType.equals(PROTOCOL_NETTY) ? ChannelType.NETTY : ChannelType.HTTP_CLIENT)) .option(ChannelOption.SO_REUSEADDR, true) .option(ChannelOption.SO_KEEPALIVE, true); @@ -203,7 +200,7 @@ private NettyMessagingTransport( .channel(NioServerSocketChannel.class) .childHandler(new NettyChannelInitializer(new NettyDefaultChannelHandlerFactory("server", this.serverChannelGroup, this.serverEventListener), sslContextServer, - protocolType == PROTOCOL_NETTY ? HANDLER_NETTY : HANDLER_HTTP_SERVER)) + protocolType.equals(PROTOCOL_NETTY) ? ChannelType.NETTY : ChannelType.HTTP_SERVER)) .option(ChannelOption.SO_BACKLOG, 128) .option(ChannelOption.SO_REUSEADDR, true) .childOption(ChannelOption.SO_KEEPALIVE, true); diff --git a/lang/java/reef-wake/wake/src/test/java/org/apache/reef/wake/test/remote/TransportHttpTest.java b/lang/java/reef-wake/wake/src/test/java/org/apache/reef/wake/test/remote/TransportHttpTest.java index ff20ce0dba..c189cb46e8 100644 --- a/lang/java/reef-wake/wake/src/test/java/org/apache/reef/wake/test/remote/TransportHttpTest.java +++ b/lang/java/reef-wake/wake/src/test/java/org/apache/reef/wake/test/remote/TransportHttpTest.java @@ -44,6 +44,8 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.logging.Level; +import static org.apache.reef.wake.remote.RemoteConfiguration.PROTOCOL_HTTP; + /** * Tests for Transport via http. @@ -77,7 +79,7 @@ public void testHttpTransportString() throws Exception { // Codec final ReceiverStage stage = new ReceiverStage<>(new ObjectSerializableCodec(), monitor, expected); - final Transport transport = tpFactory.newInstance(hostAddress, 0, stage, stage, 1, 10000, 101); + final Transport transport = tpFactory.newInstance(hostAddress, 0, stage, stage, 1, 10000, PROTOCOL_HTTP); final int port = transport.getListeningPort(); // sending side @@ -109,7 +111,7 @@ public void testHttpTransportTestEvent() throws Exception { // Codec final ReceiverStage stage = new ReceiverStage<>(new ObjectSerializableCodec(), monitor, expected); - final Transport transport = tpFactory.newInstance(hostAddress, 0, stage, stage, 1, 10000, 101); + final Transport transport = tpFactory.newInstance(hostAddress, 0, stage, stage, 1, 10000, PROTOCOL_HTTP); final int port = transport.getListeningPort(); // sending side From dc361807e6b28212da5a18053d4c95cd589d365d Mon Sep 17 00:00:00 2001 From: Kim Doyoung Date: Tue, 22 Aug 2017 14:56:29 +0900 Subject: [PATCH 10/34] added a java doc and fixed format error added a java doc and fixed format error --- .../remote/transport/netty/NettyChannelFutureListener.java | 3 +++ .../org/apache/reef/wake/remote/transport/netty/NettyLink.java | 3 ++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyChannelFutureListener.java b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyChannelFutureListener.java index 6b7a450ce2..0f5255012f 100644 --- a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyChannelFutureListener.java +++ b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyChannelFutureListener.java @@ -22,6 +22,9 @@ import io.netty.channel.ChannelFutureListener; import org.apache.reef.wake.remote.transport.LinkListener; +/** + * Future Listener used in NettyLink. + */ public final class NettyChannelFutureListener implements ChannelFutureListener { private final T message; diff --git a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyLink.java b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyLink.java index 10524f5353..b24ae0549f 100644 --- a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyLink.java +++ b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyLink.java @@ -99,7 +99,8 @@ public void write(final T message) { if (this.uri != null) { try { - final FullHttpRequest request = new DefaultFullHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.POST, uri.getRawPath()); + final FullHttpRequest request = + new DefaultFullHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.POST, uri.getRawPath()); final ByteBuf buf = Unpooled.copiedBuffer(encoder.encode(message)); request.headers() .set(HttpHeaders.Names.HOST, uri.getHost()) From abd316da07c1d9c44a0855328fc09e0d1d16635b Mon Sep 17 00:00:00 2001 From: Kim Doyoung Date: Tue, 22 Aug 2017 15:03:53 +0900 Subject: [PATCH 11/34] reverted some interface back --- .../transport/netty/AbstractNettyEventListener.java | 8 ++++---- .../wake/remote/transport/netty/LinkReference.java | 12 ++++++------ .../transport/netty/NettyChannelHandlerFactory.java | 2 +- .../remote/transport/netty/NettyEventListener.java | 2 +- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/AbstractNettyEventListener.java b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/AbstractNettyEventListener.java index 823fd3b5d8..d286d0c553 100644 --- a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/AbstractNettyEventListener.java +++ b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/AbstractNettyEventListener.java @@ -33,7 +33,7 @@ * Generic functionality for the Netty event listener. * This is a base class for client and server versions. */ -public abstract class AbstractNettyEventListener implements NettyEventListener { +abstract class AbstractNettyEventListener implements NettyEventListener { protected static final Logger LOG = Logger.getLogger(AbstractNettyEventListener.class.getName()); @@ -41,9 +41,9 @@ public abstract class AbstractNettyEventListener implements NettyEventListener { protected final EStage stage; protected EventHandler exceptionHandler; - public AbstractNettyEventListener( - final ConcurrentMap addrToLinkRefMap, - final EStage stage) { + AbstractNettyEventListener( + final ConcurrentMap addrToLinkRefMap, + final EStage stage) { this.addrToLinkRefMap = addrToLinkRefMap; this.stage = stage; } diff --git a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/LinkReference.java b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/LinkReference.java index 6afe136587..6e780b888a 100644 --- a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/LinkReference.java +++ b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/LinkReference.java @@ -26,27 +26,27 @@ * A reference for a link. * When channel became active, LinkReference is created and mapped with remote address. */ -public final class LinkReference { +final class LinkReference { private final AtomicInteger connectInProgress = new AtomicInteger(0); private Link link; - public LinkReference() { + LinkReference() { } - public LinkReference(final Link link) { + LinkReference(final Link link) { this.link = link; } - public synchronized Link getLink() { + synchronized Link getLink() { return this.link; } - public synchronized void setLink(final Link link) { + synchronized void setLink(final Link link) { this.link = link; } - public AtomicInteger getConnectInProgress() { + AtomicInteger getConnectInProgress() { return this.connectInProgress; } } diff --git a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyChannelHandlerFactory.java b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyChannelHandlerFactory.java index 665bb195a0..912a6074f0 100644 --- a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyChannelHandlerFactory.java +++ b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyChannelHandlerFactory.java @@ -24,7 +24,7 @@ /** * Factory that creates a Netty channel handler. */ -public interface NettyChannelHandlerFactory { +interface NettyChannelHandlerFactory { /** * Creates a channel inbound handler. diff --git a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyEventListener.java b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyEventListener.java index faa9e1eebb..a6a7359672 100644 --- a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyEventListener.java +++ b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyEventListener.java @@ -23,7 +23,7 @@ /** * Netty event listener. */ -public interface NettyEventListener { +interface NettyEventListener { /** * Handles the message. From 7ce5b333ab2d5e1640de8b9f0bfdbfbea4c7e7f1 Mon Sep 17 00:00:00 2001 From: Kim Doyoung Date: Tue, 22 Aug 2017 17:37:12 +0900 Subject: [PATCH 12/34] Fixed implementation to accept PROTOCOL_HTTPS --- .../reef/wake/remote/RemoteConfiguration.java | 3 ++- .../netty/NettyMessagingTransport.java | 19 ++++++++++++------- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/RemoteConfiguration.java b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/RemoteConfiguration.java index 8cc1291a4d..f979eb9e29 100644 --- a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/RemoteConfiguration.java +++ b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/RemoteConfiguration.java @@ -49,10 +49,11 @@ public final class RemoteConfiguration { /** - * Unique protocol numbers for choosing protocols. + * Unique protocol String for choosing protocols. */ public static final String PROTOCOL_NETTY = "__PROTOCOL_NETTY__"; public static final String PROTOCOL_HTTP = "__PROTOCOL_HTTP__"; + public static final String PROTOCOL_HTTPS = "__PROTOCOL_HTTPS__"; private RemoteConfiguration() { // empty diff --git a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyMessagingTransport.java b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyMessagingTransport.java index e0beace776..f3bcbed4b7 100644 --- a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyMessagingTransport.java +++ b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyMessagingTransport.java @@ -61,6 +61,7 @@ import java.util.logging.Logger; import static org.apache.reef.wake.remote.RemoteConfiguration.PROTOCOL_HTTP; +import static org.apache.reef.wake.remote.RemoteConfiguration.PROTOCOL_HTTPS; import static org.apache.reef.wake.remote.RemoteConfiguration.PROTOCOL_NETTY; import static org.apache.reef.wake.remote.transport.netty.NettyChannelInitializer.ChannelType; @@ -143,12 +144,13 @@ private NettyMessagingTransport( supportsSSL = System.getProperty("ssl") != null; - if (protocolType.equals(PROTOCOL_HTTP)) { + if (protocolType.equals(PROTOCOL_HTTPS)) { if (supportsSSL) { try { final SelfSignedCertificate ssc = new SelfSignedCertificate(); sslContextClient = SslContext.newServerContext(ssc.certificate(), ssc.privateKey()); sslContextServer = SslContext.newClientContext(); + this.uri = URI.create("https://" + hostAddress); LOG.log(Level.FINE, "SSL context created"); } catch (final Exception ex) { final RuntimeException transportException = @@ -157,15 +159,18 @@ private NettyMessagingTransport( throw transportException; } } else { - LOG.log(Level.FINE, "System property 'ssl' is not set"); - sslContextClient = null; - sslContextServer = null; + LOG.log(Level.SEVERE, "Could not find support for SSL"); + throw new TransportRuntimeException("Could not find support for SSL"); } - this.uri = URI.create((supportsSSL ? "https://" : "http://") + hostAddress); - } else { + + } else { // for HTTP and default Netty sslContextClient = null; sslContextServer = null; - this.uri = null; + if (protocolType.equals(PROTOCOL_HTTP)) { + this.uri = URI.create("http://" + hostAddress); + } else { + this.uri = null; + } } this.numberOfTries = numberOfTries; From 14121e7110023c52f629e1c6e539bb2508341746 Mon Sep 17 00:00:00 2001 From: Kim Doyoung Date: Wed, 23 Aug 2017 14:38:32 +0900 Subject: [PATCH 13/34] fixed TransportHttpTest --- .../wake/test/remote/TransportHttpTest.java | 36 ++++++++++--------- 1 file changed, 20 insertions(+), 16 deletions(-) diff --git a/lang/java/reef-wake/wake/src/test/java/org/apache/reef/wake/test/remote/TransportHttpTest.java b/lang/java/reef-wake/wake/src/test/java/org/apache/reef/wake/test/remote/TransportHttpTest.java index c189cb46e8..6449b36fc7 100644 --- a/lang/java/reef-wake/wake/src/test/java/org/apache/reef/wake/test/remote/TransportHttpTest.java +++ b/lang/java/reef-wake/wake/src/test/java/org/apache/reef/wake/test/remote/TransportHttpTest.java @@ -87,8 +87,8 @@ public void testHttpTransportString() throws Exception { new InetSocketAddress(hostAddress, port), new ObjectSerializableCodec(), new LoggingLinkListener()); - link.write(new String("hello1")); - link.write(new String("hello2")); + link.write("hello1"); + link.write("hello2"); monitor.mwait(); transport.close(); @@ -111,20 +111,24 @@ public void testHttpTransportTestEvent() throws Exception { // Codec final ReceiverStage stage = new ReceiverStage<>(new ObjectSerializableCodec(), monitor, expected); - final Transport transport = tpFactory.newInstance(hostAddress, 0, stage, stage, 1, 10000, PROTOCOL_HTTP); - final int port = transport.getListeningPort(); - - // sending side - final Link link = transport.open( - new InetSocketAddress(hostAddress, port), - new ObjectSerializableCodec(), - new LoggingLinkListener()); - link.write(new TestEvent("hello1", 0.0)); - link.write(new TestEvent("hello2", 1.0)); - monitor.mwait(); - transport.close(); - timer.close(); + try ( + final Transport transport = + tpFactory.newInstance(hostAddress, 0, stage, stage, 1, 10000, PROTOCOL_HTTP) + ) { + final int port = transport.getListeningPort(); + + // sending side + final Link link = transport.open( + new InetSocketAddress(hostAddress, port), + new ObjectSerializableCodec(), + new LoggingLinkListener()); + link.write(new TestEvent("hello1", 0.0)); + link.write(new TestEvent("hello2", 1.0)); + + monitor.mwait(); + timer.close(); + } Assert.assertEquals(expected, stage.getCount()); } @@ -134,7 +138,7 @@ class ReceiverStage implements EStage { private final Codec codec; private final Monitor monitor; private final int expected; - private AtomicInteger count = new AtomicInteger(0); + private final AtomicInteger count = new AtomicInteger(0); ReceiverStage(final Codec codec, final Monitor monitor, final int expected) { this.codec = codec; From 476ebe1953f83513505fa9df783a339a279c8c0c Mon Sep 17 00:00:00 2001 From: Kim Doyoung Date: Wed, 23 Aug 2017 16:54:31 +0900 Subject: [PATCH 14/34] Fixed String into ProtocolTypes --- .../apache/reef/wake/remote/RemoteConfiguration.java | 10 ++++++---- .../reef/wake/remote/transport/TransportFactory.java | 11 +++++++++-- .../transport/netty/MessagingTransportFactory.java | 10 +++++----- .../transport/netty/NettyMessagingTransport.java | 8 ++++---- .../reef/wake/test/remote/TransportHttpTest.java | 6 +++--- 5 files changed, 27 insertions(+), 18 deletions(-) diff --git a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/RemoteConfiguration.java b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/RemoteConfiguration.java index f979eb9e29..16df7e5600 100644 --- a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/RemoteConfiguration.java +++ b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/RemoteConfiguration.java @@ -26,6 +26,7 @@ import org.apache.reef.wake.remote.impl.DefaultTransportEStage; import org.apache.reef.wake.remote.impl.ObjectSerializableCodec; import org.apache.reef.wake.remote.impl.TransportEvent; +import org.apache.reef.wake.remote.transport.TransportFactory.ProtocolTypes; /** * Configuration options and helper methods for Wake remoting. @@ -51,9 +52,9 @@ public final class RemoteConfiguration { /** * Unique protocol String for choosing protocols. */ - public static final String PROTOCOL_NETTY = "__PROTOCOL_NETTY__"; - public static final String PROTOCOL_HTTP = "__PROTOCOL_HTTP__"; - public static final String PROTOCOL_HTTPS = "__PROTOCOL_HTTPS__"; + public static final String PROTOCOL_TCP = ProtocolTypes.TCP.name(); + public static final String PROTOCOL_HTTP = ProtocolTypes.HTTP.name(); + public static final String PROTOCOL_HTTPS = ProtocolTypes.HTTPS.name(); private RemoteConfiguration() { // empty @@ -143,8 +144,9 @@ public static final class RemoteServerStage implements Name { // Intentionally empty } diff --git a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/TransportFactory.java b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/TransportFactory.java index 18576dc053..011c9e0d37 100644 --- a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/TransportFactory.java +++ b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/TransportFactory.java @@ -31,6 +31,13 @@ @DefaultImplementation(MessagingTransportFactory.class) public interface TransportFactory { + /** + * Types of protocol used in Transport. + */ + enum ProtocolTypes { + TCP, HTTP, HTTPS + } + /** * Creates a transport. * @@ -79,7 +86,7 @@ Transport newInstance(final String hostAddress, int port, final EStage serverStage, final int numberOfTries, final int retryTimeout, - final String protocol); + final ProtocolTypes protocol); /** * Creates a transport. @@ -121,7 +128,7 @@ Transport newInstance(final String hostAddress, final int numberOfTries, final int retryTimeout, final TcpPortProvider tcpPortProvider, - final String protocol); + final ProtocolTypes protocol); } diff --git a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/MessagingTransportFactory.java b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/MessagingTransportFactory.java index bd745dfa31..c26df0702c 100644 --- a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/MessagingTransportFactory.java +++ b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/MessagingTransportFactory.java @@ -119,11 +119,11 @@ public Transport newInstance(final String hostAddress, final EStage serverStage, final int numberOfTries, final int retryTimeout, - final String protocol) { + final ProtocolTypes protocol) { try { final TcpPortProvider tcpPortProvider = TANG.newInjector().getInstance(TcpPortProvider.class); return newInstance(hostAddress, port, clientStage, - serverStage, numberOfTries, retryTimeout, tcpPortProvider); + serverStage, numberOfTries, retryTimeout, tcpPortProvider, protocol); } catch (final InjectionException e) { throw new RuntimeException(e); } @@ -150,7 +150,7 @@ public Transport newInstance(final String hostAddress, final TcpPortProvider tcpPortProvider) { return newInstance(hostAddress, port, clientStage, - serverStage, numberOfTries, retryTimeout, tcpPortProvider, RemoteConfiguration.PROTOCOL_NETTY); + serverStage, numberOfTries, retryTimeout, tcpPortProvider, ProtocolTypes.TCP); } /** @@ -173,7 +173,7 @@ public Transport newInstance(final String hostAddress, final int numberOfTries, final int retryTimeout, final TcpPortProvider tcpPortProvider, - final String protocol) { + final ProtocolTypes protocol) { final Injector injector = TANG.newInjector(); injector.bindVolatileParameter(RemoteConfiguration.HostAddress.class, hostAddress); @@ -183,7 +183,7 @@ public Transport newInstance(final String hostAddress, injector.bindVolatileParameter(RemoteConfiguration.NumberOfTries.class, numberOfTries); injector.bindVolatileParameter(RemoteConfiguration.RetryTimeout.class, retryTimeout); injector.bindVolatileInstance(TcpPortProvider.class, tcpPortProvider); - injector.bindVolatileParameter(RemoteConfiguration.Protocol.class, protocol); + injector.bindVolatileParameter(RemoteConfiguration.Protocol.class, protocol.name()); try { return injector.getInstance(NettyMessagingTransport.class); } catch (final InjectionException e) { diff --git a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyMessagingTransport.java b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyMessagingTransport.java index f3bcbed4b7..793a2e6ac0 100644 --- a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyMessagingTransport.java +++ b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyMessagingTransport.java @@ -62,7 +62,7 @@ import static org.apache.reef.wake.remote.RemoteConfiguration.PROTOCOL_HTTP; import static org.apache.reef.wake.remote.RemoteConfiguration.PROTOCOL_HTTPS; -import static org.apache.reef.wake.remote.RemoteConfiguration.PROTOCOL_NETTY; +import static org.apache.reef.wake.remote.RemoteConfiguration.PROTOCOL_TCP; import static org.apache.reef.wake.remote.transport.netty.NettyChannelInitializer.ChannelType; /** @@ -175,7 +175,7 @@ private NettyMessagingTransport( this.numberOfTries = numberOfTries; this.retryTimeout = retryTimeout; - if (protocolType.equals(PROTOCOL_NETTY)) { + if (protocolType.equals(PROTOCOL_TCP)) { this.clientEventListener = new NettyClientEventListener(this.addrToLinkRefMap, clientStage); this.serverEventListener = new NettyServerEventListener(this.addrToLinkRefMap, serverStage); } else { @@ -196,7 +196,7 @@ private NettyMessagingTransport( .channel(NioSocketChannel.class) .handler(new NettyChannelInitializer(new NettyDefaultChannelHandlerFactory("client", this.clientChannelGroup, this.clientEventListener), sslContextClient, - protocolType.equals(PROTOCOL_NETTY) ? ChannelType.NETTY : ChannelType.HTTP_CLIENT)) + protocolType.equals(PROTOCOL_TCP) ? ChannelType.NETTY : ChannelType.HTTP_CLIENT)) .option(ChannelOption.SO_REUSEADDR, true) .option(ChannelOption.SO_KEEPALIVE, true); @@ -205,7 +205,7 @@ private NettyMessagingTransport( .channel(NioServerSocketChannel.class) .childHandler(new NettyChannelInitializer(new NettyDefaultChannelHandlerFactory("server", this.serverChannelGroup, this.serverEventListener), sslContextServer, - protocolType.equals(PROTOCOL_NETTY) ? ChannelType.NETTY : ChannelType.HTTP_SERVER)) + protocolType.equals(PROTOCOL_TCP) ? ChannelType.NETTY : ChannelType.HTTP_SERVER)) .option(ChannelOption.SO_BACKLOG, 128) .option(ChannelOption.SO_REUSEADDR, true) .childOption(ChannelOption.SO_KEEPALIVE, true); diff --git a/lang/java/reef-wake/wake/src/test/java/org/apache/reef/wake/test/remote/TransportHttpTest.java b/lang/java/reef-wake/wake/src/test/java/org/apache/reef/wake/test/remote/TransportHttpTest.java index 6449b36fc7..4a9d35e588 100644 --- a/lang/java/reef-wake/wake/src/test/java/org/apache/reef/wake/test/remote/TransportHttpTest.java +++ b/lang/java/reef-wake/wake/src/test/java/org/apache/reef/wake/test/remote/TransportHttpTest.java @@ -44,7 +44,7 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.logging.Level; -import static org.apache.reef.wake.remote.RemoteConfiguration.PROTOCOL_HTTP; +import static org.apache.reef.wake.remote.transport.TransportFactory.ProtocolTypes; /** @@ -79,7 +79,7 @@ public void testHttpTransportString() throws Exception { // Codec final ReceiverStage stage = new ReceiverStage<>(new ObjectSerializableCodec(), monitor, expected); - final Transport transport = tpFactory.newInstance(hostAddress, 0, stage, stage, 1, 10000, PROTOCOL_HTTP); + final Transport transport = tpFactory.newInstance(hostAddress, 0, stage, stage, 1, 10000, ProtocolTypes.HTTP); final int port = transport.getListeningPort(); // sending side @@ -114,7 +114,7 @@ public void testHttpTransportTestEvent() throws Exception { try ( final Transport transport = - tpFactory.newInstance(hostAddress, 0, stage, stage, 1, 10000, PROTOCOL_HTTP) + tpFactory.newInstance(hostAddress, 0, stage, stage, 1, 10000, ProtocolTypes.HTTP) ) { final int port = transport.getListeningPort(); From 2befc1afb857e3b1124071f58a4ab1884879125b Mon Sep 17 00:00:00 2001 From: Kim Doyoung Date: Wed, 23 Aug 2017 17:16:32 +0900 Subject: [PATCH 15/34] Fixed reflexing reviews --- .../apache/reef/wake/remote/transport/TransportFactory.java | 5 +++-- .../remote/transport/netty/NettyChannelFutureListener.java | 2 +- .../wake/remote/transport/netty/NettyChannelInitializer.java | 4 ++-- .../remote/transport/netty/NettyHttpClientEventListener.java | 5 +++-- .../remote/transport/netty/NettyHttpServerEventListener.java | 3 ++- .../wake/remote/transport/netty/NettyMessagingTransport.java | 4 ++-- .../org/apache/reef/wake/test/remote/TransportHttpTest.java | 3 --- 7 files changed, 13 insertions(+), 13 deletions(-) diff --git a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/TransportFactory.java b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/TransportFactory.java index 011c9e0d37..06edd9a468 100644 --- a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/TransportFactory.java +++ b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/TransportFactory.java @@ -81,7 +81,8 @@ Transport newInstance(final String hostAddress, int port, * @param protocol protocol to use * @return transport */ - Transport newInstance(final String hostAddress, int port, + Transport newInstance(final String hostAddress, + final int port, final EStage clientStage, final EStage serverStage, final int numberOfTries, @@ -122,7 +123,7 @@ Transport newInstance(final String hostAddress, * @return transport */ Transport newInstance(final String hostAddress, - int port, + final int port, final EStage clientStage, final EStage serverStage, final int numberOfTries, diff --git a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyChannelFutureListener.java b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyChannelFutureListener.java index 0f5255012f..1fba96f26b 100644 --- a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyChannelFutureListener.java +++ b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyChannelFutureListener.java @@ -28,7 +28,7 @@ public final class NettyChannelFutureListener implements ChannelFutureListener { private final T message; - private LinkListener listener; + private final LinkListener listener; NettyChannelFutureListener(final T message, final LinkListener listener) { this.message = message; diff --git a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyChannelInitializer.java b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyChannelInitializer.java index 7a72e4f015..019964524e 100644 --- a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyChannelInitializer.java +++ b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyChannelInitializer.java @@ -34,7 +34,7 @@ class NettyChannelInitializer extends ChannelInitializer { /* Types for initiating channel */ public enum ChannelType { - NETTY, HTTP_SERVER, HTTP_CLIENT + TCP, HTTP_SERVER, HTTP_CLIENT } /** @@ -65,7 +65,7 @@ public enum ChannelType { @Override protected void initChannel(final SocketChannel ch) throws Exception { switch (this.type) { - case NETTY: + case TCP: ch.pipeline() .addLast("frameDecoder", new LengthFieldBasedFrameDecoder(MAXFRAMELENGTH, 0, 4, 0, 4)) .addLast("bytesDecoder", new ByteArrayDecoder()) diff --git a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyHttpClientEventListener.java b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyHttpClientEventListener.java index 5daaf2a267..ddc55f9477 100644 --- a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyHttpClientEventListener.java +++ b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyHttpClientEventListener.java @@ -34,7 +34,7 @@ import java.util.logging.Logger; /** - * A Netty event listener for server. + * A Netty event listener for client. */ final class NettyHttpClientEventListener extends AbstractNettyEventListener { @@ -56,7 +56,7 @@ public void channelRead(final ChannelHandlerContext ctx, final Object msg) { } else if (msg instanceof HttpContent) { final HttpContent httpContent = (HttpContent) msg; final ByteBuf content = httpContent.content(); - if (content.isReadable() && LOG.isLoggable(Level.FINEST)) { + if (LOG.isLoggable(Level.FINEST) && content.isReadable()) { buf.append("CONTENT: ").append(content.toString(CharsetUtil.UTF_8)).append("\r\n"); } @@ -67,6 +67,7 @@ public void channelRead(final ChannelHandlerContext ctx, final Object msg) { if (LOG.isLoggable(Level.FINEST)) { LOG.log(Level.FINEST, "MessageEvent: local: {0} remote: {1} :: {2}", new Object[]{ channel.localAddress(), channel.remoteAddress(), buf}); + buf.setLength(0); // clearing the buffer } if (message.length > 0) { diff --git a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyHttpServerEventListener.java b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyHttpServerEventListener.java index 377f15858e..920075c26b 100644 --- a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyHttpServerEventListener.java +++ b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyHttpServerEventListener.java @@ -78,7 +78,7 @@ public void channelRead(final ChannelHandlerContext ctx, final Object msg) { this.httpRequest = request; if (!headers.isEmpty()) { - for (final Map.Entry h: headers) { + for (final Map.Entry h : headers) { final CharSequence key = h.getKey(); final CharSequence value = h.getValue(); buf.append("HEADER: ").append(key).append(" = ").append(value).append("\r\n"); @@ -109,6 +109,7 @@ public void channelRead(final ChannelHandlerContext ctx, final Object msg) { } } buf.append("\r\n"); + buf.setLength(0); // clearing the buffer } final Channel channel = ctx.channel(); diff --git a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyMessagingTransport.java b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyMessagingTransport.java index 793a2e6ac0..22b9ae9af5 100644 --- a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyMessagingTransport.java +++ b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyMessagingTransport.java @@ -196,7 +196,7 @@ private NettyMessagingTransport( .channel(NioSocketChannel.class) .handler(new NettyChannelInitializer(new NettyDefaultChannelHandlerFactory("client", this.clientChannelGroup, this.clientEventListener), sslContextClient, - protocolType.equals(PROTOCOL_TCP) ? ChannelType.NETTY : ChannelType.HTTP_CLIENT)) + protocolType.equals(PROTOCOL_TCP) ? ChannelType.TCP : ChannelType.HTTP_CLIENT)) .option(ChannelOption.SO_REUSEADDR, true) .option(ChannelOption.SO_KEEPALIVE, true); @@ -205,7 +205,7 @@ private NettyMessagingTransport( .channel(NioServerSocketChannel.class) .childHandler(new NettyChannelInitializer(new NettyDefaultChannelHandlerFactory("server", this.serverChannelGroup, this.serverEventListener), sslContextServer, - protocolType.equals(PROTOCOL_TCP) ? ChannelType.NETTY : ChannelType.HTTP_SERVER)) + protocolType.equals(PROTOCOL_TCP) ? ChannelType.TCP : ChannelType.HTTP_SERVER)) .option(ChannelOption.SO_BACKLOG, 128) .option(ChannelOption.SO_REUSEADDR, true) .childOption(ChannelOption.SO_KEEPALIVE, true); diff --git a/lang/java/reef-wake/wake/src/test/java/org/apache/reef/wake/test/remote/TransportHttpTest.java b/lang/java/reef-wake/wake/src/test/java/org/apache/reef/wake/test/remote/TransportHttpTest.java index 4a9d35e588..34875baa66 100644 --- a/lang/java/reef-wake/wake/src/test/java/org/apache/reef/wake/test/remote/TransportHttpTest.java +++ b/lang/java/reef-wake/wake/src/test/java/org/apache/reef/wake/test/remote/TransportHttpTest.java @@ -60,13 +60,11 @@ public TransportHttpTest() throws InjectionException { this.tpFactory = injector.getInstance(MessagingTransportFactory.class); } - private static final String LOG_PREFIX = "TEST "; @Rule public TestName name = new TestName(); @Test public void testHttpTransportString() throws Exception { - System.out.println(LOG_PREFIX + name.getMethodName()); LoggingUtils.setLoggingLevel(Level.INFO); final Monitor monitor = new Monitor(); @@ -99,7 +97,6 @@ public void testHttpTransportString() throws Exception { @Test public void testHttpTransportTestEvent() throws Exception { - System.out.println(LOG_PREFIX + name.getMethodName()); LoggingUtils.setLoggingLevel(Level.INFO); final Monitor monitor = new Monitor(); From df5fea582b73535725c12a09e7286f1d255e7900 Mon Sep 17 00:00:00 2001 From: Kim Doyoung Date: Wed, 23 Aug 2017 19:37:13 +0900 Subject: [PATCH 16/34] implemented NettyLinkFactory and its default class --- .../netty/NettyDefaultLinkFactory.java | 58 ++++++++ .../remote/transport/netty/NettyHttpLink.java | 136 ++++++++++++++++++ .../netty/NettyHttpServerEventListener.java | 8 +- .../remote/transport/netty/NettyLink.java | 51 +------ .../transport/netty/NettyLinkFactory.java | 49 +++++++ .../netty/NettyMessagingTransport.java | 4 +- .../netty/NettyServerEventListener.java | 6 +- 7 files changed, 259 insertions(+), 53 deletions(-) create mode 100644 lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyDefaultLinkFactory.java create mode 100644 lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyHttpLink.java create mode 100644 lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyLinkFactory.java diff --git a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyDefaultLinkFactory.java b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyDefaultLinkFactory.java new file mode 100644 index 0000000000..6af318201b --- /dev/null +++ b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyDefaultLinkFactory.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.reef.wake.remote.transport.netty; + +import io.netty.channel.Channel; +import org.apache.reef.wake.remote.Encoder; +import org.apache.reef.wake.remote.transport.Link; +import org.apache.reef.wake.remote.transport.LinkListener; + +import java.net.URI; + +/** + * Factory that creates a NettyLink. + */ +public final class NettyDefaultLinkFactory implements NettyLinkFactory { + + private final URI uri; + + NettyDefaultLinkFactory() { + this(null); + } + + NettyDefaultLinkFactory(final URI uri){ + this.uri = uri; + } + + @Override + public Link newInstance(final Channel channel, final Encoder encoder) { + return uri == null ? + new NettyLink(channel, encoder) : + new NettyHttpLink(channel, encoder, uri); + } + + @Override + public Link newInstance(final Channel channel, + final Encoder encoder, + final LinkListener listener) { + return uri == null ? + new NettyLink(channel, encoder, listener) : + new NettyHttpLink(channel, encoder, listener, uri); + } +} diff --git a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyHttpLink.java b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyHttpLink.java new file mode 100644 index 0000000000..c7a5be8282 --- /dev/null +++ b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyHttpLink.java @@ -0,0 +1,136 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.reef.wake.remote.transport.netty; + +import io.netty.buffer.ByteBuf; +import io.netty.buffer.Unpooled; +import io.netty.channel.Channel; +import io.netty.channel.ChannelFuture; +import io.netty.handler.codec.http.*; +import org.apache.reef.wake.remote.Encoder; +import org.apache.reef.wake.remote.transport.Link; +import org.apache.reef.wake.remote.transport.LinkListener; + +import java.net.SocketAddress; +import java.net.URI; +import java.util.logging.Level; +import java.util.logging.Logger; + +/** + * Link implementation with Netty. + * + * If you set a {@code LinkListener}, it keeps message until writeAndFlush operation completes + * and notifies whether the sent message transferred successfully through the listener. + */ +public class NettyHttpLink implements Link { + + public static final int INT_SIZE = Integer.SIZE / Byte.SIZE; + + private static final Logger LOG = Logger.getLogger(NettyHttpLink.class.getName()); + + private final Channel channel; + private final Encoder encoder; + private final LinkListener listener; + private final URI uri; + + /** + * Constructs a link. + * + * @param channel the channel + * @param encoder the encoder + * @param uri the URI + */ + public NettyHttpLink(final Channel channel, + final Encoder encoder, + final URI uri) { + this(channel, encoder, null, uri); + } + + /** + * Constructs a link. + * + * @param channel the channel + * @param encoder the encoder + * @param listener the link listener + * @param uri the URI + */ + public NettyHttpLink( + final Channel channel, + final Encoder encoder, + final LinkListener listener, + final URI uri) { + this.channel = channel; + this.encoder = encoder; + this.listener = listener; + this.uri = uri; + } + /** + * Writes the message to this link. + * + * @param message the message + */ + @Override + public void write(final T message) { + LOG.log(Level.FINEST, "write {0} :: {1}", new Object[] {channel, message}); + try { + final FullHttpRequest request = + new DefaultFullHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.POST, uri.getRawPath()); + final ByteBuf buf = Unpooled.copiedBuffer(encoder.encode(message)); + request.headers() + .set(HttpHeaders.Names.HOST, uri.getHost()) + .set(HttpHeaders.Names.CONNECTION, HttpHeaders.Values.KEEP_ALIVE) + .set(HttpHeaders.Names.ACCEPT_ENCODING, HttpHeaders.Values.GZIP) + .set(HttpHeaders.Names.CONTENT_TYPE, "application/wake-transport") + .set(HttpHeaders.Names.CONTENT_LENGTH, buf.readableBytes()); + request.content().clear().writeBytes(buf); + final ChannelFuture future = channel.writeAndFlush(request); + future.sync(); + if (listener != null) { + future.addListener(new NettyChannelFutureListener<>(message, listener)); + } + } catch (final InterruptedException ex) { + LOG.log(Level.SEVERE, "Cannot send request to " + uri.getHost(), ex); + } + } + + /** + * Gets a local address of the link. + * + * @return a local socket address + */ + @Override + public SocketAddress getLocalAddress() { + return channel.localAddress(); + } + + /** + * Gets a remote address of the link. + * + * @return a remote socket address + */ + @Override + public SocketAddress getRemoteAddress() { + return channel.remoteAddress(); + } + + @Override + public String toString() { + return "NettyLink: " + channel; // Channel has good .toString() implementation + } +} diff --git a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyHttpServerEventListener.java b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyHttpServerEventListener.java index 920075c26b..092e090dfb 100644 --- a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyHttpServerEventListener.java +++ b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyHttpServerEventListener.java @@ -42,6 +42,7 @@ final class NettyHttpServerEventListener extends AbstractNettyEventListener { private HttpRequest httpRequest; private final StringBuilder buf = new StringBuilder(); private final URI uri; + private final NettyLinkFactory linkFactory; NettyHttpServerEventListener( final ConcurrentMap addrToLinkRefMap, @@ -49,6 +50,7 @@ final class NettyHttpServerEventListener extends AbstractNettyEventListener { final URI uri) { super(addrToLinkRefMap, stage); this.uri = uri; + this.linkFactory = new NettyDefaultLinkFactory<>(uri); } @@ -61,8 +63,8 @@ public void channelActive(final ChannelHandlerContext ctx) { } this.addrToLinkRefMap.putIfAbsent( - channel.remoteAddress(), new LinkReference(new NettyLink<>( - channel, new ByteCodec(), new LoggingLinkListener(), uri))); + channel.remoteAddress(), new LinkReference(linkFactory.newInstance( + channel, new ByteCodec(), new LoggingLinkListener()))); LOG.log(Level.FINER, "Add connected channel ref: {0}", this.addrToLinkRefMap.get(channel.remoteAddress())); @@ -144,7 +146,7 @@ private static void appendDecoderResult(final StringBuilder buf, final HttpObjec @Override protected TransportEvent getTransportEvent(final byte[] message, final Channel channel) { - return new TransportEvent(message, new NettyLink<>(channel, new ByteEncoder())); + return new TransportEvent(message, linkFactory.newInstance(channel, new ByteEncoder())); } @Override diff --git a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyLink.java b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyLink.java index b24ae0549f..61c30c8d11 100644 --- a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyLink.java +++ b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyLink.java @@ -18,17 +18,14 @@ */ package org.apache.reef.wake.remote.transport.netty; -import io.netty.buffer.ByteBuf; import io.netty.buffer.Unpooled; import io.netty.channel.Channel; import io.netty.channel.ChannelFuture; -import io.netty.handler.codec.http.*; import org.apache.reef.wake.remote.Encoder; import org.apache.reef.wake.remote.transport.Link; import org.apache.reef.wake.remote.transport.LinkListener; import java.net.SocketAddress; -import java.net.URI; import java.util.logging.Level; import java.util.logging.Logger; @@ -47,7 +44,6 @@ public class NettyLink implements Link { private final Channel channel; private final Encoder encoder; private final LinkListener listener; - private final URI uri; /** * Constructs a link. @@ -67,27 +63,11 @@ public NettyLink(final Channel channel, final Encoder encoder) { * @param listener the link listener */ public NettyLink(final Channel channel, final Encoder encoder, final LinkListener listener) { - this(channel, encoder, listener, null); - } - - /** - * Constructs a link. - * - * @param channel the channel - * @param encoder the encoder - * @param listener the link listener - * @param uri the URI - */ - public NettyLink( - final Channel channel, - final Encoder encoder, - final LinkListener listener, - final URI uri) { this.channel = channel; this.encoder = encoder; this.listener = listener; - this.uri = uri; } + /** * Writes the message to this link. * @@ -96,32 +76,9 @@ public NettyLink( @Override public void write(final T message) { LOG.log(Level.FINEST, "write {0} :: {1}", new Object[] {channel, message}); - - if (this.uri != null) { - try { - final FullHttpRequest request = - new DefaultFullHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.POST, uri.getRawPath()); - final ByteBuf buf = Unpooled.copiedBuffer(encoder.encode(message)); - request.headers() - .set(HttpHeaders.Names.HOST, uri.getHost()) - .set(HttpHeaders.Names.CONNECTION, HttpHeaders.Values.KEEP_ALIVE) - .set(HttpHeaders.Names.ACCEPT_ENCODING, HttpHeaders.Values.GZIP) - .set(HttpHeaders.Names.CONTENT_TYPE, "application/wake-transport") - .set(HttpHeaders.Names.CONTENT_LENGTH, buf.readableBytes()); - request.content().clear().writeBytes(buf); - final ChannelFuture future = channel.writeAndFlush(request); - future.sync(); - if (listener != null) { - future.addListener(new NettyChannelFutureListener<>(message, listener)); - } - } catch (final InterruptedException ex) { - LOG.log(Level.SEVERE, "Cannot send request to " + uri.getHost(), ex); - } - } else { - final ChannelFuture future = channel.writeAndFlush(Unpooled.wrappedBuffer(encoder.encode(message))); - if (listener != null) { - future.addListener(new NettyChannelFutureListener<>(message, listener)); - } + final ChannelFuture future = channel.writeAndFlush(Unpooled.wrappedBuffer(encoder.encode(message))); + if (listener != null) { + future.addListener(new NettyChannelFutureListener<>(message, listener)); } } diff --git a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyLinkFactory.java b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyLinkFactory.java new file mode 100644 index 0000000000..5f6cc076f1 --- /dev/null +++ b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyLinkFactory.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.reef.wake.remote.transport.netty; + +import io.netty.channel.Channel; +import org.apache.reef.tang.annotations.DefaultImplementation; +import org.apache.reef.wake.remote.Encoder; +import org.apache.reef.wake.remote.transport.Link; +import org.apache.reef.wake.remote.transport.LinkListener; + +/** + * Factory that creates a NettyLink. + */ +@DefaultImplementation(NettyDefaultLinkFactory.class) +public interface NettyLinkFactory { + + /** + * Creates a NettyLink. + * @param channel the channel + * @param encoder the encoder + */ + Link newInstance(final Channel channel, final Encoder encoder); + + /** + * Creates a NettyLink. + * @param channel the channel + * @param encoder the encoder + * @param listener the listener + */ + Link newInstance(final Channel channel, + final Encoder encoder, + final LinkListener listener); +} diff --git a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyMessagingTransport.java b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyMessagingTransport.java index 22b9ae9af5..6100e7c118 100644 --- a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyMessagingTransport.java +++ b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyMessagingTransport.java @@ -362,7 +362,9 @@ public Link open(final SocketAddress remoteAddr, final Encoder connectFuture = this.clientBootstrap.connect(remoteAddr); connectFuture.syncUninterruptibly(); - link = new NettyLink<>(connectFuture.channel(), encoder, listener, this.uri); + final NettyLinkFactory linkFactory = new NettyDefaultLinkFactory<>(uri); + + link = linkFactory.newInstance(connectFuture.channel(), encoder, listener); linkRef.setLink(link); synchronized (flag) { diff --git a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyServerEventListener.java b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyServerEventListener.java index d4d8f888bc..9657f0dcd9 100644 --- a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyServerEventListener.java +++ b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyServerEventListener.java @@ -33,6 +33,8 @@ */ final class NettyServerEventListener extends AbstractNettyEventListener { + private NettyLinkFactory linkFactory = new NettyDefaultLinkFactory(); + NettyServerEventListener( final ConcurrentMap addrToLinkRefMap, final EStage stage) { @@ -49,7 +51,7 @@ public void channelActive(final ChannelHandlerContext ctx) { } this.addrToLinkRefMap.putIfAbsent( - channel.remoteAddress(), new LinkReference(new NettyLink<>( + channel.remoteAddress(), new LinkReference(linkFactory.newInstance( channel, new ByteCodec(), new LoggingLinkListener()))); LOG.log(Level.FINER, "Add connected channel ref: {0}", this.addrToLinkRefMap.get(channel.remoteAddress())); @@ -58,7 +60,7 @@ public void channelActive(final ChannelHandlerContext ctx) { @Override protected TransportEvent getTransportEvent(final byte[] message, final Channel channel) { - return new TransportEvent(message, new NettyLink<>(channel, new ByteEncoder())); + return new TransportEvent(message, linkFactory.newInstance(channel, new ByteEncoder())); } @Override From 985ad1a5aba0347b88c5d783df68b6454a4cd57b Mon Sep 17 00:00:00 2001 From: nhne Date: Thu, 24 Aug 2017 16:55:13 +0900 Subject: [PATCH 17/34] added NettyHttpLinkFactory --- .../netty/NettyDefaultLinkFactory.java | 20 ++------ .../remote/transport/netty/NettyHttpLink.java | 6 ++- .../transport/netty/NettyHttpLinkFactory.java | 50 +++++++++++++++++++ .../netty/NettyHttpServerEventListener.java | 2 +- .../transport/netty/NettyLinkFactory.java | 4 +- .../netty/NettyMessagingTransport.java | 3 +- 6 files changed, 62 insertions(+), 23 deletions(-) create mode 100644 lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyHttpLinkFactory.java diff --git a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyDefaultLinkFactory.java b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyDefaultLinkFactory.java index 6af318201b..370fa5b104 100644 --- a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyDefaultLinkFactory.java +++ b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyDefaultLinkFactory.java @@ -23,36 +23,22 @@ import org.apache.reef.wake.remote.transport.Link; import org.apache.reef.wake.remote.transport.LinkListener; -import java.net.URI; - /** * Factory that creates a NettyLink. */ public final class NettyDefaultLinkFactory implements NettyLinkFactory { - private final URI uri; - - NettyDefaultLinkFactory() { - this(null); - } - - NettyDefaultLinkFactory(final URI uri){ - this.uri = uri; - } + NettyDefaultLinkFactory() {} @Override public Link newInstance(final Channel channel, final Encoder encoder) { - return uri == null ? - new NettyLink(channel, encoder) : - new NettyHttpLink(channel, encoder, uri); + return new NettyLink(channel, encoder); } @Override public Link newInstance(final Channel channel, final Encoder encoder, final LinkListener listener) { - return uri == null ? - new NettyLink(channel, encoder, listener) : - new NettyHttpLink(channel, encoder, listener, uri); + return new NettyLink(channel, encoder, listener); } } diff --git a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyHttpLink.java b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyHttpLink.java index c7a5be8282..2dc38e78af 100644 --- a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyHttpLink.java +++ b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyHttpLink.java @@ -38,7 +38,7 @@ * If you set a {@code LinkListener}, it keeps message until writeAndFlush operation completes * and notifies whether the sent message transferred successfully through the listener. */ -public class NettyHttpLink implements Link { +public final class NettyHttpLink implements Link { public static final int INT_SIZE = Integer.SIZE / Byte.SIZE; @@ -131,6 +131,8 @@ public SocketAddress getRemoteAddress() { @Override public String toString() { - return "NettyLink: " + channel; // Channel has good .toString() implementation + return new StringBuilder() + .append("NettyLink: ") + .append(channel).toString(); // Channel has good .toString() implementation } } diff --git a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyHttpLinkFactory.java b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyHttpLinkFactory.java new file mode 100644 index 0000000000..3491b96b8b --- /dev/null +++ b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyHttpLinkFactory.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.reef.wake.remote.transport.netty; + +import io.netty.channel.Channel; +import org.apache.reef.wake.remote.Encoder; +import org.apache.reef.wake.remote.transport.Link; +import org.apache.reef.wake.remote.transport.LinkListener; + +import java.net.URI; + +/** + * Factory that creates a NettyHttpLink. + */ +public final class NettyHttpLinkFactory implements NettyLinkFactory { + + private final URI uri; + + NettyHttpLinkFactory(final URI uri){ + this.uri = uri; + } + + @Override + public Link newInstance(final Channel channel, final Encoder encoder) { + return new NettyHttpLink(channel, encoder, uri); + } + + @Override + public Link newInstance(final Channel channel, + final Encoder encoder, + final LinkListener listener) { + return new NettyHttpLink(channel, encoder, listener, uri); + } +} diff --git a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyHttpServerEventListener.java b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyHttpServerEventListener.java index 092e090dfb..70618e2de0 100644 --- a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyHttpServerEventListener.java +++ b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyHttpServerEventListener.java @@ -50,7 +50,7 @@ final class NettyHttpServerEventListener extends AbstractNettyEventListener { final URI uri) { super(addrToLinkRefMap, stage); this.uri = uri; - this.linkFactory = new NettyDefaultLinkFactory<>(uri); + this.linkFactory = new NettyHttpLinkFactory<>(uri); } diff --git a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyLinkFactory.java b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyLinkFactory.java index 5f6cc076f1..94c62a5891 100644 --- a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyLinkFactory.java +++ b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyLinkFactory.java @@ -44,6 +44,6 @@ public interface NettyLinkFactory { * @param listener the listener */ Link newInstance(final Channel channel, - final Encoder encoder, - final LinkListener listener); + final Encoder encoder, + final LinkListener listener); } diff --git a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyMessagingTransport.java b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyMessagingTransport.java index 6100e7c118..c026ee3769 100644 --- a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyMessagingTransport.java +++ b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyMessagingTransport.java @@ -362,7 +362,8 @@ public Link open(final SocketAddress remoteAddr, final Encoder connectFuture = this.clientBootstrap.connect(remoteAddr); connectFuture.syncUninterruptibly(); - final NettyLinkFactory linkFactory = new NettyDefaultLinkFactory<>(uri); + final NettyLinkFactory linkFactory = + uri == null ? new NettyDefaultLinkFactory<>() : new NettyHttpLinkFactory(uri); link = linkFactory.newInstance(connectFuture.channel(), encoder, listener); linkRef.setLink(link); From 8fd67f6045a1db0d70e2cb56a472e0c067d32c81 Mon Sep 17 00:00:00 2001 From: nhne Date: Thu, 24 Aug 2017 18:34:11 +0900 Subject: [PATCH 18/34] excluded HTTPS related codes --- .../reef/wake/remote/RemoteConfiguration.java | 2 +- .../remote/transport/TransportFactory.java | 2 +- .../netty/NettyMessagingTransport.java | 39 ++++--------------- 3 files changed, 10 insertions(+), 33 deletions(-) diff --git a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/RemoteConfiguration.java b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/RemoteConfiguration.java index 16df7e5600..80ba41f6ae 100644 --- a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/RemoteConfiguration.java +++ b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/RemoteConfiguration.java @@ -54,7 +54,7 @@ public final class RemoteConfiguration { */ public static final String PROTOCOL_TCP = ProtocolTypes.TCP.name(); public static final String PROTOCOL_HTTP = ProtocolTypes.HTTP.name(); - public static final String PROTOCOL_HTTPS = ProtocolTypes.HTTPS.name(); + //TODO[JIRA REEF-1871] Implement HTTPS as protocol private RemoteConfiguration() { // empty diff --git a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/TransportFactory.java b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/TransportFactory.java index 06edd9a468..d70240338f 100644 --- a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/TransportFactory.java +++ b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/TransportFactory.java @@ -35,7 +35,7 @@ public interface TransportFactory { * Types of protocol used in Transport. */ enum ProtocolTypes { - TCP, HTTP, HTTPS + TCP, HTTP } /** diff --git a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyMessagingTransport.java b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyMessagingTransport.java index c026ee3769..e6cabee6b9 100644 --- a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyMessagingTransport.java +++ b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyMessagingTransport.java @@ -31,7 +31,6 @@ import io.netty.channel.socket.nio.NioServerSocketChannel; import io.netty.channel.socket.nio.NioSocketChannel; import io.netty.handler.ssl.SslContext; -import io.netty.handler.ssl.util.SelfSignedCertificate; import io.netty.util.concurrent.Future; import io.netty.util.concurrent.GlobalEventExecutor; import org.apache.reef.tang.annotations.Parameter; @@ -61,7 +60,6 @@ import java.util.logging.Logger; import static org.apache.reef.wake.remote.RemoteConfiguration.PROTOCOL_HTTP; -import static org.apache.reef.wake.remote.RemoteConfiguration.PROTOCOL_HTTPS; import static org.apache.reef.wake.remote.RemoteConfiguration.PROTOCOL_TCP; import static org.apache.reef.wake.remote.transport.netty.NettyChannelInitializer.ChannelType; @@ -102,7 +100,6 @@ public final class NettyMessagingTransport implements Transport { private final AbstractNettyEventListener clientEventListener; private final AbstractNettyEventListener serverEventListener; - private final boolean supportsSSL; private final URI uri; private final int numberOfTries; @@ -142,35 +139,15 @@ private NettyMessagingTransport( final SslContext sslContextClient; final SslContext sslContextServer; - supportsSSL = System.getProperty("ssl") != null; - - if (protocolType.equals(PROTOCOL_HTTPS)) { - if (supportsSSL) { - try { - final SelfSignedCertificate ssc = new SelfSignedCertificate(); - sslContextClient = SslContext.newServerContext(ssc.certificate(), ssc.privateKey()); - sslContextServer = SslContext.newClientContext(); - this.uri = URI.create("https://" + hostAddress); - LOG.log(Level.FINE, "SSL context created"); - } catch (final Exception ex) { - final RuntimeException transportException = - new TransportRuntimeException("Could not create SSL Context", ex); - LOG.log(Level.SEVERE, "Could not create SSL Context", ex); - throw transportException; - } - } else { - LOG.log(Level.SEVERE, "Could not find support for SSL"); - throw new TransportRuntimeException("Could not find support for SSL"); - } + //TODO[JIRA REEF-1871] Implement HTTPS with sslContext. - } else { // for HTTP and default Netty - sslContextClient = null; - sslContextServer = null; - if (protocolType.equals(PROTOCOL_HTTP)) { - this.uri = URI.create("http://" + hostAddress); - } else { - this.uri = null; - } + // for HTTP and default Netty + sslContextClient = null; + sslContextServer = null; + if (protocolType.equals(PROTOCOL_HTTP)) { + this.uri = URI.create("http://" + hostAddress); + } else { + this.uri = null; } this.numberOfTries = numberOfTries; From 37a9adafba8d8dead84e603f38d8ee9094f8c077 Mon Sep 17 00:00:00 2001 From: nhne Date: Fri, 1 Sep 2017 16:46:52 +0900 Subject: [PATCH 19/34] removed ssl part --- .../transport/netty/NettyChannelInitializer.java | 14 -------------- .../transport/netty/NettyMessagingTransport.java | 10 ++-------- 2 files changed, 2 insertions(+), 22 deletions(-) diff --git a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyChannelInitializer.java b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyChannelInitializer.java index 019964524e..7e68029c11 100644 --- a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyChannelInitializer.java +++ b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyChannelInitializer.java @@ -25,7 +25,6 @@ import io.netty.handler.codec.bytes.ByteArrayDecoder; import io.netty.handler.codec.bytes.ByteArrayEncoder; import io.netty.handler.codec.http.*; -import io.netty.handler.ssl.SslContext; /** * Netty channel initializer for Transport. @@ -43,11 +42,6 @@ public enum ChannelType { public static final int MAXFRAMELENGTH = 10 * 1024 * 1024; private final NettyChannelHandlerFactory handlerFactory; - /** - * sslContext contains ssl context of the machine. used only for HTTP. - */ - private final SslContext sslContext; - /** * Type of channel whether it is netty or http client or http server. */ @@ -55,10 +49,8 @@ public enum ChannelType { NettyChannelInitializer( final NettyChannelHandlerFactory handlerFactory, - final SslContext sslContext, final ChannelType type) { this.handlerFactory = handlerFactory; - this.sslContext = sslContext; this.type = type; } @@ -75,9 +67,6 @@ protected void initChannel(final SocketChannel ch) throws Exception { .addLast("handler", handlerFactory.createChannelInboundHandler()); break; case HTTP_SERVER: - if (sslContext != null) { - ch.pipeline().addLast(sslContext.newHandler(ch.alloc())); - } ch.pipeline() .addLast("codec", new HttpServerCodec()) .addLast("requestDecoder", new HttpRequestDecoder()) @@ -85,9 +74,6 @@ protected void initChannel(final SocketChannel ch) throws Exception { .addLast("handler", handlerFactory.createChannelInboundHandler()); break; case HTTP_CLIENT: - if (sslContext != null) { - ch.pipeline().addLast(sslContext.newHandler(ch.alloc())); - } ch.pipeline() .addLast("codec", new HttpClientCodec()) .addLast("decompressor", new HttpContentDecompressor()) diff --git a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyMessagingTransport.java b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyMessagingTransport.java index e6cabee6b9..9e82f86fd5 100644 --- a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyMessagingTransport.java +++ b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyMessagingTransport.java @@ -30,7 +30,6 @@ import io.netty.channel.nio.NioEventLoopGroup; import io.netty.channel.socket.nio.NioServerSocketChannel; import io.netty.channel.socket.nio.NioSocketChannel; -import io.netty.handler.ssl.SslContext; import io.netty.util.concurrent.Future; import io.netty.util.concurrent.GlobalEventExecutor; import org.apache.reef.tang.annotations.Parameter; @@ -136,14 +135,9 @@ private NettyMessagingTransport( final String host = UNKNOWN_HOST_NAME.equals(hostAddress) ? localAddressProvider.getLocalAddress() : hostAddress; - final SslContext sslContextClient; - final SslContext sslContextServer; - //TODO[JIRA REEF-1871] Implement HTTPS with sslContext. // for HTTP and default Netty - sslContextClient = null; - sslContextServer = null; if (protocolType.equals(PROTOCOL_HTTP)) { this.uri = URI.create("http://" + hostAddress); } else { @@ -172,7 +166,7 @@ private NettyMessagingTransport( this.clientBootstrap.group(this.clientWorkerGroup) .channel(NioSocketChannel.class) .handler(new NettyChannelInitializer(new NettyDefaultChannelHandlerFactory("client", - this.clientChannelGroup, this.clientEventListener), sslContextClient, + this.clientChannelGroup, this.clientEventListener), protocolType.equals(PROTOCOL_TCP) ? ChannelType.TCP : ChannelType.HTTP_CLIENT)) .option(ChannelOption.SO_REUSEADDR, true) .option(ChannelOption.SO_KEEPALIVE, true); @@ -181,7 +175,7 @@ private NettyMessagingTransport( this.serverBootstrap.group(this.serverBossGroup, this.serverWorkerGroup) .channel(NioServerSocketChannel.class) .childHandler(new NettyChannelInitializer(new NettyDefaultChannelHandlerFactory("server", - this.serverChannelGroup, this.serverEventListener), sslContextServer, + this.serverChannelGroup, this.serverEventListener), protocolType.equals(PROTOCOL_TCP) ? ChannelType.TCP : ChannelType.HTTP_SERVER)) .option(ChannelOption.SO_BACKLOG, 128) .option(ChannelOption.SO_REUSEADDR, true) From 817fa0ef5a922e6a1e2fb768fc4b38731da8983b Mon Sep 17 00:00:00 2001 From: nhne Date: Fri, 1 Sep 2017 19:37:33 +0900 Subject: [PATCH 20/34] changed constant String into enum and altered NettyChannelInitializer --- .../reef/wake/remote/RemoteConfiguration.java | 14 ++--- .../remote/transport/TransportFactory.java | 6 +-- .../netty/MessagingTransportFactory.java | 8 +-- .../netty/NettyChannelInitializer.java | 51 +++++++++++-------- .../netty/NettyMessagingTransport.java | 17 +++---- .../wake/test/remote/TransportHttpTest.java | 7 ++- 6 files changed, 48 insertions(+), 55 deletions(-) diff --git a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/RemoteConfiguration.java b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/RemoteConfiguration.java index 80ba41f6ae..a73ef2a4a4 100644 --- a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/RemoteConfiguration.java +++ b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/RemoteConfiguration.java @@ -26,7 +26,7 @@ import org.apache.reef.wake.remote.impl.DefaultTransportEStage; import org.apache.reef.wake.remote.impl.ObjectSerializableCodec; import org.apache.reef.wake.remote.impl.TransportEvent; -import org.apache.reef.wake.remote.transport.TransportFactory.ProtocolTypes; +import org.apache.reef.wake.remote.transport.TransportFactory.ProtocolType; /** * Configuration options and helper methods for Wake remoting. @@ -48,14 +48,6 @@ public final class RemoteConfiguration { public static final long REMOTE_CONNECTION_RETRY_TIMEOUT = WakeParameters.REMOTE_EXECUTOR_SHUTDOWN_TIMEOUT / (REMOTE_CONNECTION_NUMBER_OF_RETRIES + 1); - - /** - * Unique protocol String for choosing protocols. - */ - public static final String PROTOCOL_TCP = ProtocolTypes.TCP.name(); - public static final String PROTOCOL_HTTP = ProtocolTypes.HTTP.name(); - //TODO[JIRA REEF-1871] Implement HTTPS as protocol - private RemoteConfiguration() { // empty } @@ -144,10 +136,10 @@ public static final class RemoteServerStage implements Name { + public static final class Protocol implements Name { // Intentionally empty } } diff --git a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/TransportFactory.java b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/TransportFactory.java index d70240338f..3d802955a9 100644 --- a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/TransportFactory.java +++ b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/TransportFactory.java @@ -34,7 +34,7 @@ public interface TransportFactory { /** * Types of protocol used in Transport. */ - enum ProtocolTypes { + enum ProtocolType { TCP, HTTP } @@ -87,7 +87,7 @@ Transport newInstance(final String hostAddress, final EStage serverStage, final int numberOfTries, final int retryTimeout, - final ProtocolTypes protocol); + final ProtocolType protocol); /** * Creates a transport. @@ -129,7 +129,7 @@ Transport newInstance(final String hostAddress, final int numberOfTries, final int retryTimeout, final TcpPortProvider tcpPortProvider, - final ProtocolTypes protocol); + final ProtocolType protocol); } diff --git a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/MessagingTransportFactory.java b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/MessagingTransportFactory.java index c26df0702c..0dc33f0f7a 100644 --- a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/MessagingTransportFactory.java +++ b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/MessagingTransportFactory.java @@ -119,7 +119,7 @@ public Transport newInstance(final String hostAddress, final EStage serverStage, final int numberOfTries, final int retryTimeout, - final ProtocolTypes protocol) { + final ProtocolType protocol) { try { final TcpPortProvider tcpPortProvider = TANG.newInjector().getInstance(TcpPortProvider.class); return newInstance(hostAddress, port, clientStage, @@ -150,7 +150,7 @@ public Transport newInstance(final String hostAddress, final TcpPortProvider tcpPortProvider) { return newInstance(hostAddress, port, clientStage, - serverStage, numberOfTries, retryTimeout, tcpPortProvider, ProtocolTypes.TCP); + serverStage, numberOfTries, retryTimeout, tcpPortProvider, ProtocolType.TCP); } /** @@ -173,7 +173,7 @@ public Transport newInstance(final String hostAddress, final int numberOfTries, final int retryTimeout, final TcpPortProvider tcpPortProvider, - final ProtocolTypes protocol) { + final ProtocolType protocol) { final Injector injector = TANG.newInjector(); injector.bindVolatileParameter(RemoteConfiguration.HostAddress.class, hostAddress); @@ -183,7 +183,7 @@ public Transport newInstance(final String hostAddress, injector.bindVolatileParameter(RemoteConfiguration.NumberOfTries.class, numberOfTries); injector.bindVolatileParameter(RemoteConfiguration.RetryTimeout.class, retryTimeout); injector.bindVolatileInstance(TcpPortProvider.class, tcpPortProvider); - injector.bindVolatileParameter(RemoteConfiguration.Protocol.class, protocol.name()); + injector.bindVolatileParameter(RemoteConfiguration.Protocol.class, protocol); try { return injector.getInstance(NettyMessagingTransport.class); } catch (final InjectionException e) { diff --git a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyChannelInitializer.java b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyChannelInitializer.java index 7e68029c11..59a309a4f5 100644 --- a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyChannelInitializer.java +++ b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyChannelInitializer.java @@ -25,17 +25,14 @@ import io.netty.handler.codec.bytes.ByteArrayDecoder; import io.netty.handler.codec.bytes.ByteArrayEncoder; import io.netty.handler.codec.http.*; +import org.apache.reef.wake.remote.transport.TransportFactory.ProtocolType; + /** * Netty channel initializer for Transport. */ class NettyChannelInitializer extends ChannelInitializer { - /* Types for initiating channel */ - public enum ChannelType { - TCP, HTTP_SERVER, HTTP_CLIENT - } - /** * the buffer size of the frame decoder. */ @@ -43,20 +40,29 @@ public enum ChannelType { private final NettyChannelHandlerFactory handlerFactory; /** - * Type of channel whether it is netty or http client or http server. + * Type of protocol channel use. */ - private final ChannelType type; + private final ProtocolType protocolType; + private final boolean isServer; + + NettyChannelInitializer( + final NettyChannelHandlerFactory handlerFactory, + final ProtocolType protocol) { + this(handlerFactory, protocol, false); + } NettyChannelInitializer( final NettyChannelHandlerFactory handlerFactory, - final ChannelType type) { + final ProtocolType protocol, + final boolean isServer) { this.handlerFactory = handlerFactory; - this.type = type; + this.protocolType = protocol; + this.isServer = isServer; } @Override protected void initChannel(final SocketChannel ch) throws Exception { - switch (this.type) { + switch (this.protocolType) { case TCP: ch.pipeline() .addLast("frameDecoder", new LengthFieldBasedFrameDecoder(MAXFRAMELENGTH, 0, 4, 0, 4)) @@ -66,18 +72,19 @@ protected void initChannel(final SocketChannel ch) throws Exception { .addLast("chunker", new ChunkedReadWriteHandler()) .addLast("handler", handlerFactory.createChannelInboundHandler()); break; - case HTTP_SERVER: - ch.pipeline() - .addLast("codec", new HttpServerCodec()) - .addLast("requestDecoder", new HttpRequestDecoder()) - .addLast("responseEncoder", new HttpResponseEncoder()) - .addLast("handler", handlerFactory.createChannelInboundHandler()); - break; - case HTTP_CLIENT: - ch.pipeline() - .addLast("codec", new HttpClientCodec()) - .addLast("decompressor", new HttpContentDecompressor()) - .addLast("handler", handlerFactory.createChannelInboundHandler()); + case HTTP: + if(isServer) { + ch.pipeline() + .addLast("codec", new HttpServerCodec()) + .addLast("requestDecoder", new HttpRequestDecoder()) + .addLast("responseEncoder", new HttpResponseEncoder()) + .addLast("handler", handlerFactory.createChannelInboundHandler()); + } else { + ch.pipeline() + .addLast("codec", new HttpClientCodec()) + .addLast("decompressor", new HttpContentDecompressor()) + .addLast("handler", handlerFactory.createChannelInboundHandler()); + } break; default: throw new IllegalArgumentException("Invalid type of channel"); diff --git a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyMessagingTransport.java b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyMessagingTransport.java index f73a2599c2..e8c19c35fe 100644 --- a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyMessagingTransport.java +++ b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyMessagingTransport.java @@ -45,6 +45,7 @@ import org.apache.reef.wake.remote.transport.Link; import org.apache.reef.wake.remote.transport.LinkListener; import org.apache.reef.wake.remote.transport.Transport; +import org.apache.reef.wake.remote.transport.TransportFactory.ProtocolType; import org.apache.reef.wake.remote.transport.exception.TransportRuntimeException; import javax.inject.Inject; @@ -58,10 +59,6 @@ import java.util.logging.Level; import java.util.logging.Logger; -import static org.apache.reef.wake.remote.RemoteConfiguration.PROTOCOL_HTTP; -import static org.apache.reef.wake.remote.RemoteConfiguration.PROTOCOL_TCP; -import static org.apache.reef.wake.remote.transport.netty.NettyChannelInitializer.ChannelType; - /** * Messaging transport implementation with Netty and Http. */ @@ -126,7 +123,7 @@ private NettyMessagingTransport( @Parameter(RemoteConfiguration.RetryTimeout.class) final int retryTimeout, final TcpPortProvider tcpPortProvider, final LocalAddressProvider localAddressProvider, - @Parameter(RemoteConfiguration.Protocol.class) final String protocolType) { + @Parameter(RemoteConfiguration.Protocol.class) final ProtocolType protocolType) { int p = port; if (p < 0) { @@ -138,7 +135,7 @@ private NettyMessagingTransport( //TODO[JIRA REEF-1871] Implement HTTPS with sslContext. // for HTTP and default Netty - if (protocolType.equals(PROTOCOL_HTTP)) { + if (protocolType == ProtocolType.HTTP) { this.uri = URI.create("http://" + hostAddress); } else { this.uri = null; @@ -146,7 +143,7 @@ private NettyMessagingTransport( this.numberOfTries = numberOfTries; this.retryTimeout = retryTimeout; - if (protocolType.equals(PROTOCOL_TCP)) { + if (protocolType == ProtocolType.TCP) { this.clientEventListener = new NettyClientEventListener(this.addrToLinkRefMap, clientStage); this.serverEventListener = new NettyServerEventListener(this.addrToLinkRefMap, serverStage); } else { @@ -166,8 +163,7 @@ private NettyMessagingTransport( this.clientBootstrap.group(this.clientWorkerGroup) .channel(NioSocketChannel.class) .handler(new NettyChannelInitializer(new NettyDefaultChannelHandlerFactory("client", - this.clientChannelGroup, this.clientEventListener), - protocolType.equals(PROTOCOL_TCP) ? ChannelType.TCP : ChannelType.HTTP_CLIENT)) + this.clientChannelGroup, this.clientEventListener), protocolType)) .option(ChannelOption.SO_REUSEADDR, true) .option(ChannelOption.SO_KEEPALIVE, true); @@ -175,8 +171,7 @@ private NettyMessagingTransport( this.serverBootstrap.group(this.serverBossGroup, this.serverWorkerGroup) .channel(NioServerSocketChannel.class) .childHandler(new NettyChannelInitializer(new NettyDefaultChannelHandlerFactory("server", - this.serverChannelGroup, this.serverEventListener), - protocolType.equals(PROTOCOL_TCP) ? ChannelType.TCP : ChannelType.HTTP_SERVER)) + this.serverChannelGroup, this.serverEventListener), protocolType, true)) .option(ChannelOption.SO_BACKLOG, 128) .option(ChannelOption.SO_REUSEADDR, true) .childOption(ChannelOption.SO_KEEPALIVE, true); diff --git a/lang/java/reef-wake/wake/src/test/java/org/apache/reef/wake/test/remote/TransportHttpTest.java b/lang/java/reef-wake/wake/src/test/java/org/apache/reef/wake/test/remote/TransportHttpTest.java index 34875baa66..e87140bdf4 100644 --- a/lang/java/reef-wake/wake/src/test/java/org/apache/reef/wake/test/remote/TransportHttpTest.java +++ b/lang/java/reef-wake/wake/src/test/java/org/apache/reef/wake/test/remote/TransportHttpTest.java @@ -44,8 +44,6 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.logging.Level; -import static org.apache.reef.wake.remote.transport.TransportFactory.ProtocolTypes; - /** * Tests for Transport via http. @@ -77,7 +75,8 @@ public void testHttpTransportString() throws Exception { // Codec final ReceiverStage stage = new ReceiverStage<>(new ObjectSerializableCodec(), monitor, expected); - final Transport transport = tpFactory.newInstance(hostAddress, 0, stage, stage, 1, 10000, ProtocolTypes.HTTP); + final Transport transport = + tpFactory.newInstance(hostAddress, 0, stage, stage, 1, 10000, TransportFactory.ProtocolType.HTTP); final int port = transport.getListeningPort(); // sending side @@ -111,7 +110,7 @@ public void testHttpTransportTestEvent() throws Exception { try ( final Transport transport = - tpFactory.newInstance(hostAddress, 0, stage, stage, 1, 10000, ProtocolTypes.HTTP) + tpFactory.newInstance(hostAddress, 0, stage, stage, 1, 10000, TransportFactory.ProtocolType.HTTP) ) { final int port = transport.getListeningPort(); From 4990972ee23a616f944e5bd4c6fc997c2283feb3 Mon Sep 17 00:00:00 2001 From: Kim Doyoung Date: Wed, 6 Sep 2017 17:23:35 +0900 Subject: [PATCH 21/34] changed TransportFactory to use protocol as constructor parameter --- .../remote/transport/TransportFactory.java | 42 ------------- .../netty/MessagingTransportFactory.java | 62 +++---------------- 2 files changed, 7 insertions(+), 97 deletions(-) diff --git a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/TransportFactory.java b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/TransportFactory.java index 3d802955a9..0ea562f44e 100644 --- a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/TransportFactory.java +++ b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/TransportFactory.java @@ -69,26 +69,6 @@ Transport newInstance(final String hostAddress, int port, final int numberOfTries, final int retryTimeout); - /** - * Creates a transport. - * - * @param hostAddress a host address - * @param port a listening port - * @param clientStage a transport client-side stage - * @param serverStage a transport server-side stage - * @param numberOfTries the number of retries for connection - * @param retryTimeout retry timeout - * @param protocol protocol to use - * @return transport - */ - Transport newInstance(final String hostAddress, - final int port, - final EStage clientStage, - final EStage serverStage, - final int numberOfTries, - final int retryTimeout, - final ProtocolType protocol); - /** * Creates a transport. * @@ -109,27 +89,5 @@ Transport newInstance(final String hostAddress, final int retryTimeout, final TcpPortProvider tcpPortProvider); - /** - * Creates a transport. - * - * @param hostAddress a host address - * @param port a listening port - * @param clientStage a transport client-side stage - * @param serverStage a transport server-side stage - * @param numberOfTries the number of retries for connection - * @param retryTimeout retry timeout - * @param tcpPortProvider tcpPortProvider - * @param protocol protocol to use - * @return transport - */ - Transport newInstance(final String hostAddress, - final int port, - final EStage clientStage, - final EStage serverStage, - final int numberOfTries, - final int retryTimeout, - final TcpPortProvider tcpPortProvider, - final ProtocolType protocol); - } diff --git a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/MessagingTransportFactory.java b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/MessagingTransportFactory.java index 0dc33f0f7a..b638612d82 100644 --- a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/MessagingTransportFactory.java +++ b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/MessagingTransportFactory.java @@ -20,6 +20,7 @@ import org.apache.reef.tang.Injector; import org.apache.reef.tang.Tang; +import org.apache.reef.tang.annotations.Parameter; import org.apache.reef.tang.exceptions.InjectionException; import org.apache.reef.wake.EStage; import org.apache.reef.wake.EventHandler; @@ -39,11 +40,14 @@ public final class MessagingTransportFactory implements TransportFactory { private final String localAddress; + private final ProtocolType protocol; private static final Tang TANG = Tang.Factory.getTang(); @Inject - private MessagingTransportFactory(final LocalAddressProvider localAddressProvider) { + private MessagingTransportFactory(final LocalAddressProvider localAddressProvider, + @Parameter(RemoteConfiguration.Protocol.class) final ProtocolType protocolType) { this.localAddress = localAddressProvider.getLocalAddress(); + this.protocol = protocolType; } /** @@ -65,6 +69,7 @@ public Transport newInstance(final int port, injector.bindVolatileParameter(RemoteConfiguration.Port.class, port); injector.bindVolatileParameter(RemoteConfiguration.RemoteClientStage.class, new SyncStage<>(clientHandler)); injector.bindVolatileParameter(RemoteConfiguration.RemoteServerStage.class, new SyncStage<>(serverHandler)); + injector.bindVolatileParameter(RemoteConfiguration.Protocol.class, this.protocol); final Transport transport; try { @@ -102,33 +107,6 @@ public Transport newInstance(final String hostAddress, } } - /** - * Creates a transport. - * - * @param hostAddress a host address - * @param port a listening port - * @param clientStage a client stage - * @param serverStage a server stage - * @param numberOfTries a number of tries - * @param retryTimeout a timeout for retry - */ - @Override - public Transport newInstance(final String hostAddress, - final int port, - final EStage clientStage, - final EStage serverStage, - final int numberOfTries, - final int retryTimeout, - final ProtocolType protocol) { - try { - final TcpPortProvider tcpPortProvider = TANG.newInjector().getInstance(TcpPortProvider.class); - return newInstance(hostAddress, port, clientStage, - serverStage, numberOfTries, retryTimeout, tcpPortProvider, protocol); - } catch (final InjectionException e) { - throw new RuntimeException(e); - } - } - /** * Creates a transport. * @@ -149,32 +127,6 @@ public Transport newInstance(final String hostAddress, final int retryTimeout, final TcpPortProvider tcpPortProvider) { - return newInstance(hostAddress, port, clientStage, - serverStage, numberOfTries, retryTimeout, tcpPortProvider, ProtocolType.TCP); - } - - /** - * Creates a transport. - * - * @param hostAddress a host address - * @param port a listening port - * @param clientStage a client stage - * @param serverStage a server stage - * @param numberOfTries a number of tries - * @param retryTimeout a timeout for retry - * @param tcpPortProvider a provider for TCP port - * @param protocol a protocol to use - */ - @Override - public Transport newInstance(final String hostAddress, - final int port, - final EStage clientStage, - final EStage serverStage, - final int numberOfTries, - final int retryTimeout, - final TcpPortProvider tcpPortProvider, - final ProtocolType protocol) { - final Injector injector = TANG.newInjector(); injector.bindVolatileParameter(RemoteConfiguration.HostAddress.class, hostAddress); injector.bindVolatileParameter(RemoteConfiguration.Port.class, port); @@ -183,7 +135,7 @@ public Transport newInstance(final String hostAddress, injector.bindVolatileParameter(RemoteConfiguration.NumberOfTries.class, numberOfTries); injector.bindVolatileParameter(RemoteConfiguration.RetryTimeout.class, retryTimeout); injector.bindVolatileInstance(TcpPortProvider.class, tcpPortProvider); - injector.bindVolatileParameter(RemoteConfiguration.Protocol.class, protocol); + injector.bindVolatileParameter(RemoteConfiguration.Protocol.class, this.protocol); try { return injector.getInstance(NettyMessagingTransport.class); } catch (final InjectionException e) { From 9e9260a5cf4e819d0a29d818406b654eee803bf4 Mon Sep 17 00:00:00 2001 From: Kim Doyoung Date: Wed, 6 Sep 2017 18:01:59 +0900 Subject: [PATCH 22/34] refactored TransportTest and merged TransportHttpTest into TransportTest --- .../wake/test/remote/TransportHttpTest.java | 164 ------------------ .../reef/wake/test/remote/TransportTest.java | 39 ++++- 2 files changed, 34 insertions(+), 169 deletions(-) delete mode 100644 lang/java/reef-wake/wake/src/test/java/org/apache/reef/wake/test/remote/TransportHttpTest.java diff --git a/lang/java/reef-wake/wake/src/test/java/org/apache/reef/wake/test/remote/TransportHttpTest.java b/lang/java/reef-wake/wake/src/test/java/org/apache/reef/wake/test/remote/TransportHttpTest.java deleted file mode 100644 index e87140bdf4..0000000000 --- a/lang/java/reef-wake/wake/src/test/java/org/apache/reef/wake/test/remote/TransportHttpTest.java +++ /dev/null @@ -1,164 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.reef.wake.test.remote; - -import org.apache.reef.tang.Injector; -import org.apache.reef.tang.Tang; -import org.apache.reef.tang.exceptions.InjectionException; -import org.apache.reef.wake.EStage; -import org.apache.reef.wake.impl.LoggingUtils; -import org.apache.reef.wake.impl.TimerStage; -import org.apache.reef.wake.remote.Codec; -import org.apache.reef.wake.remote.address.LocalAddressProvider; -import org.apache.reef.wake.remote.impl.ObjectSerializableCodec; -import org.apache.reef.wake.remote.impl.TransportEvent; -import org.apache.reef.wake.remote.transport.Link; -import org.apache.reef.wake.remote.transport.Transport; -import org.apache.reef.wake.remote.transport.TransportFactory; -import org.apache.reef.wake.remote.transport.netty.MessagingTransportFactory; -import org.apache.reef.wake.remote.transport.netty.LoggingLinkListener; -import org.apache.reef.wake.test.util.Monitor; -import org.apache.reef.wake.test.util.TimeoutHandler; -import org.junit.Assert; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TestName; - -import java.net.InetSocketAddress; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.logging.Level; - - -/** - * Tests for Transport via http. - */ -public class TransportHttpTest { - private final LocalAddressProvider localAddressProvider; - private final TransportFactory tpFactory; - - public TransportHttpTest() throws InjectionException { - final Injector injector = Tang.Factory.getTang().newInjector(); - this.localAddressProvider = injector.getInstance(LocalAddressProvider.class); - this.tpFactory = injector.getInstance(MessagingTransportFactory.class); - } - - @Rule - public TestName name = new TestName(); - - @Test - public void testHttpTransportString() throws Exception { - LoggingUtils.setLoggingLevel(Level.INFO); - - final Monitor monitor = new Monitor(); - final TimerStage timer = new TimerStage(new TimeoutHandler(monitor), 2000, 2000); - - final int expected = 2; - - final String hostAddress = localAddressProvider.getLocalAddress(); - - // Codec - final ReceiverStage stage = - new ReceiverStage<>(new ObjectSerializableCodec(), monitor, expected); - final Transport transport = - tpFactory.newInstance(hostAddress, 0, stage, stage, 1, 10000, TransportFactory.ProtocolType.HTTP); - final int port = transport.getListeningPort(); - - // sending side - final Link link = transport.open( - new InetSocketAddress(hostAddress, port), - new ObjectSerializableCodec(), - new LoggingLinkListener()); - link.write("hello1"); - link.write("hello2"); - - monitor.mwait(); - transport.close(); - timer.close(); - - Assert.assertEquals(expected, stage.getCount()); - } - - @Test - public void testHttpTransportTestEvent() throws Exception { - LoggingUtils.setLoggingLevel(Level.INFO); - - final Monitor monitor = new Monitor(); - final TimerStage timer = new TimerStage(new TimeoutHandler(monitor), 2000, 2000); - - final int expected = 2; - final String hostAddress = localAddressProvider.getLocalAddress(); - - // Codec - final ReceiverStage stage = - new ReceiverStage<>(new ObjectSerializableCodec(), monitor, expected); - - try ( - final Transport transport = - tpFactory.newInstance(hostAddress, 0, stage, stage, 1, 10000, TransportFactory.ProtocolType.HTTP) - ) { - final int port = transport.getListeningPort(); - - // sending side - final Link link = transport.open( - new InetSocketAddress(hostAddress, port), - new ObjectSerializableCodec(), - new LoggingLinkListener()); - link.write(new TestEvent("hello1", 0.0)); - link.write(new TestEvent("hello2", 1.0)); - - monitor.mwait(); - timer.close(); - } - - Assert.assertEquals(expected, stage.getCount()); - } - - class ReceiverStage implements EStage { - - private final Codec codec; - private final Monitor monitor; - private final int expected; - private final AtomicInteger count = new AtomicInteger(0); - - ReceiverStage(final Codec codec, final Monitor monitor, final int expected) { - this.codec = codec; - this.monitor = monitor; - this.expected = expected; - } - - int getCount() { - return count.get(); - } - - @Override - public void onNext(final TransportEvent value) { - codec.decode(value.getData()); - - if (count.incrementAndGet() == expected) { - monitor.mnotify(); - } - } - - @Override - public void close() throws Exception { - } - - } - -} diff --git a/lang/java/reef-wake/wake/src/test/java/org/apache/reef/wake/test/remote/TransportTest.java b/lang/java/reef-wake/wake/src/test/java/org/apache/reef/wake/test/remote/TransportTest.java index 5d5d4d7cde..2bdeb595ea 100644 --- a/lang/java/reef-wake/wake/src/test/java/org/apache/reef/wake/test/remote/TransportTest.java +++ b/lang/java/reef-wake/wake/src/test/java/org/apache/reef/wake/test/remote/TransportTest.java @@ -19,12 +19,14 @@ package org.apache.reef.wake.test.remote; import org.apache.reef.tang.Injector; +import org.apache.reef.tang.JavaConfigurationBuilder; import org.apache.reef.tang.Tang; import org.apache.reef.tang.exceptions.InjectionException; import org.apache.reef.wake.EStage; import org.apache.reef.wake.impl.LoggingUtils; import org.apache.reef.wake.impl.TimerStage; import org.apache.reef.wake.remote.Codec; +import org.apache.reef.wake.remote.RemoteConfiguration; import org.apache.reef.wake.remote.address.LocalAddressProvider; import org.apache.reef.wake.remote.impl.ObjectSerializableCodec; import org.apache.reef.wake.remote.impl.TransportEvent; @@ -32,6 +34,7 @@ import org.apache.reef.wake.remote.transport.Transport; import org.apache.reef.wake.remote.transport.netty.LoggingLinkListener; import org.apache.reef.wake.remote.transport.TransportFactory; +import org.apache.reef.wake.remote.transport.netty.MessagingTransportFactory; import org.apache.reef.wake.test.util.Monitor; import org.apache.reef.wake.test.util.TimeoutHandler; import org.junit.Assert; @@ -49,12 +52,20 @@ */ public class TransportTest { private final LocalAddressProvider localAddressProvider; - private final TransportFactory tpFactory; + private final TransportFactory tpTcpFactory; + private final TransportFactory tpHttpFactory; public TransportTest() throws InjectionException { - final Injector injector = Tang.Factory.getTang().newInjector(); + final Tang tang = Tang.Factory.getTang(); + + final Injector injector = tang.newInjector(); this.localAddressProvider = injector.getInstance(LocalAddressProvider.class); - this.tpFactory = injector.getInstance(TransportFactory.class); + this.tpTcpFactory = injector.getInstance(TransportFactory.class); + + //for set protocol to HTTP + final JavaConfigurationBuilder jcb = tang.newConfigurationBuilder(); + jcb.bindNamedParameter(RemoteConfiguration.Protocol.class, TransportFactory.ProtocolType.HTTP.name()); + this.tpHttpFactory = tang.newInjector(jcb.build()).getInstance(MessagingTransportFactory.class); } private static final String LOG_PREFIX = "TEST "; @@ -63,6 +74,25 @@ public TransportTest() throws InjectionException { @Test public void testTransportString() throws Exception { + transportString(tpTcpFactory); + } + + @Test + public void testTransportTestEvent() throws Exception { + transportTestEvent(tpTcpFactory); + } + + @Test + public void testHttpTransportString() throws Exception { + transportString(tpHttpFactory); + } + + @Test + public void testHttpTransportTestEvent() throws Exception { + transportTestEvent(tpHttpFactory); + } + + private void transportString(final TransportFactory tpFactory) throws Exception{ System.out.println(LOG_PREFIX + name.getMethodName()); LoggingUtils.setLoggingLevel(Level.INFO); @@ -93,8 +123,7 @@ public void testTransportString() throws Exception { Assert.assertEquals(expected, stage.getCount()); } - @Test - public void testTransportTestEvent() throws Exception { + private void transportTestEvent(final TransportFactory tpFactory) throws Exception{ System.out.println(LOG_PREFIX + name.getMethodName()); LoggingUtils.setLoggingLevel(Level.INFO); From 29344b6a6d1258a5127803f7ca4814dc5f80740c Mon Sep 17 00:00:00 2001 From: Kim Doyoung Date: Thu, 14 Sep 2017 14:07:42 +0900 Subject: [PATCH 23/34] merged RemoteMangerTestHttp into RemoteManagerTest --- .../wake/test/remote/RemoteManagerTest.java | 112 +++- .../test/remote/RemoteManagerTestHttp.java | 490 ------------------ 2 files changed, 91 insertions(+), 511 deletions(-) delete mode 100644 lang/java/reef-wake/wake/src/test/java/org/apache/reef/wake/test/remote/RemoteManagerTestHttp.java diff --git a/lang/java/reef-wake/wake/src/test/java/org/apache/reef/wake/test/remote/RemoteManagerTest.java b/lang/java/reef-wake/wake/src/test/java/org/apache/reef/wake/test/remote/RemoteManagerTest.java index 08426c0796..ee2115f91f 100644 --- a/lang/java/reef-wake/wake/src/test/java/org/apache/reef/wake/test/remote/RemoteManagerTest.java +++ b/lang/java/reef-wake/wake/src/test/java/org/apache/reef/wake/test/remote/RemoteManagerTest.java @@ -19,6 +19,7 @@ package org.apache.reef.wake.test.remote; import org.apache.reef.tang.Injector; +import org.apache.reef.tang.JavaConfigurationBuilder; import org.apache.reef.tang.Tang; import org.apache.reef.tang.exceptions.InjectionException; import org.apache.reef.wake.EventHandler; @@ -31,6 +32,8 @@ import org.apache.reef.wake.remote.impl.MultiCodec; import org.apache.reef.wake.remote.impl.ObjectSerializableCodec; import org.apache.reef.wake.remote.ports.TcpPortProvider; +import org.apache.reef.wake.remote.transport.TransportFactory; +import org.apache.reef.wake.remote.transport.netty.MessagingTransportFactory; import org.apache.reef.wake.test.util.Monitor; import org.apache.reef.wake.test.util.TimeoutHandler; import org.junit.Assert; @@ -55,12 +58,19 @@ public class RemoteManagerTest { private final LocalAddressProvider localAddressProvider; - private final RemoteManagerFactory remoteManagerFactory; + private final RemoteManagerFactory remoteManagerFactoryTcp; + private final RemoteManagerFactory remoteManagerFactoryHttp; public RemoteManagerTest() throws InjectionException { - final Injector injector = Tang.Factory.getTang().newInjector(); + final Tang tang = Tang.Factory.getTang(); + final Injector injector = tang.newInjector(); this.localAddressProvider = injector.getInstance(LocalAddressProvider.class); - this.remoteManagerFactory = injector.getInstance(RemoteManagerFactory.class); + this.remoteManagerFactoryTcp = injector.getInstance(RemoteManagerFactory.class); + + final JavaConfigurationBuilder builder = Tang.Factory.getTang().newConfigurationBuilder(); + builder.bindImplementation(TransportFactory.class, MessagingTransportFactory.class); + builder.bindNamedParameter(RemoteConfiguration.Protocol.class, "HTTP"); + this.remoteManagerFactoryHttp = tang.newInjector(builder.build()).getInstance(RemoteManagerFactory.class); } @Rule @@ -70,6 +80,65 @@ public RemoteManagerTest() throws InjectionException { @Test public void testRemoteManagerTest() throws Exception { + remoteManagerTest(remoteManagerFactoryTcp); + } + + @Test + public void testRemoteManagerConnectionRetryTest() throws Exception { + remoteManagerConnectionRetryTest(remoteManagerFactoryTcp); + } + + @Test + public void testRemoteManagerConnectionRetryWithMultipleSenderTest() throws Exception { + remoteManagerConnectionRetryWithMultipleSenderTest(remoteManagerFactoryTcp); + } + + @Test + public void testRemoteManagerOrderingGuaranteeTest() throws Exception { + remoteManagerOrderingGuaranteeTest(remoteManagerFactoryTcp); + } + + @Test + public void testRemoteManagerPBufTest() throws Exception { + remoteManagerPBufTest(remoteManagerFactoryTcp); + } + + @Test + public void testRemoteManagerExceptionTest() { + remoteManagerExceptionTest(remoteManagerFactoryTcp); + } + + @Test + public void testHttpRemoteManagerTest() throws Exception { + remoteManagerTest(remoteManagerFactoryHttp); + } + + @Test + public void testHttpRemoteManagerConnectionRetryTest() throws Exception { + remoteManagerConnectionRetryTest(remoteManagerFactoryHttp); + } + + @Test + public void testHttpRemoteManagerConnectionRetryWithMultipleSenderTest() throws Exception { + remoteManagerConnectionRetryWithMultipleSenderTest(remoteManagerFactoryHttp); + } + + @Test + public void testHttpRemoteManagerOrderingGuaranteeTest() throws Exception { + remoteManagerOrderingGuaranteeTest(remoteManagerFactoryHttp); + } + + @Test + public void testHttpRemoteManagerPBufTest() throws Exception { + remoteManagerPBufTest(remoteManagerFactoryHttp); + } + + @Test + public void testHttpRemoteManagerExceptionTest() { + remoteManagerExceptionTest(remoteManagerFactoryHttp); + } + + private void remoteManagerTest(final RemoteManagerFactory remoteManagerFactory) throws Exception { System.out.println(LOG_PREFIX + name.getMethodName()); LoggingUtils.setLoggingLevel(Level.INFO); @@ -85,7 +154,7 @@ public void testRemoteManagerTest() throws Exception { final String hostAddress = localAddressProvider.getLocalAddress(); - final RemoteManager rm = this.remoteManagerFactory.getInstance( + final RemoteManager rm = remoteManagerFactory.getInstance( "name", hostAddress, 0, codec, new LoggingEventHandler(), false, 3, 10000, localAddressProvider, Tang.Factory.getTang().newInjector().getInstance(TcpPortProvider.class)); @@ -114,17 +183,18 @@ public void testRemoteManagerTest() throws Exception { timer.close(); } - @Test - public void testRemoteManagerConnectionRetryTest() throws Exception { + private void remoteManagerConnectionRetryTest(final RemoteManagerFactory remoteManagerFactory) throws Exception { final ExecutorService smExecutor = Executors.newFixedThreadPool(1); final ExecutorService rmExecutor = Executors.newFixedThreadPool(1); - final RemoteManager sendingManager = getTestRemoteManager("sender", 9020, 3, 2000); + final RemoteManager sendingManager = + getTestRemoteManager(remoteManagerFactory, "sender", 9020, 3, 2000); final Future smFuture = smExecutor.submit(new SendingRemoteManagerThread(sendingManager, 9010, 20000)); Thread.sleep(1000); - final RemoteManager receivingManager = getTestRemoteManager("receiver", 9010, 1, 2000); + final RemoteManager receivingManager = + getTestRemoteManager(remoteManagerFactory, "receiver", 9010, 1, 2000); final Future rmFuture = rmExecutor.submit(new ReceivingRemoteManagerThread(receivingManager, 20000, 1, 2)); final int smCnt = smFuture.get(); @@ -137,14 +207,15 @@ public void testRemoteManagerConnectionRetryTest() throws Exception { Assert.assertEquals(2, rmCnt); } - @Test - public void testRemoteManagerConnectionRetryWithMultipleSenderTest() throws Exception { + private void remoteManagerConnectionRetryWithMultipleSenderTest( + final RemoteManagerFactory remoteManagerFactory) throws Exception { final int numOfSenderThreads = 5; final ExecutorService smExecutor = Executors.newFixedThreadPool(numOfSenderThreads); final ExecutorService rmExecutor = Executors.newFixedThreadPool(1); final ArrayList> smFutures = new ArrayList<>(numOfSenderThreads); - final RemoteManager sendingManager = getTestRemoteManager("sender", 9030, 3, 5000); + final RemoteManager sendingManager = + getTestRemoteManager(remoteManagerFactory, "sender", 9030, 3, 5000); for (int i = 0; i < numOfSenderThreads; i++) { smFutures.add(smExecutor.submit(new SendingRemoteManagerThread(sendingManager, 9010, 20000))); @@ -152,7 +223,8 @@ public void testRemoteManagerConnectionRetryWithMultipleSenderTest() throws Exce Thread.sleep(2000); - final RemoteManager receivingManager = getTestRemoteManager("receiver", 9010, 1, 2000); + final RemoteManager receivingManager = + getTestRemoteManager(remoteManagerFactory, "receiver", 9010, 1, 2000); final Future receivingFuture = rmExecutor.submit(new ReceivingRemoteManagerThread(receivingManager, 20000, numOfSenderThreads, 2)); @@ -171,8 +243,7 @@ public void testRemoteManagerConnectionRetryWithMultipleSenderTest() throws Exce Assert.assertEquals(2 * numOfSenderThreads, rmCnt); } - @Test - public void testRemoteManagerOrderingGuaranteeTest() throws Exception { + private void remoteManagerOrderingGuaranteeTest(final RemoteManagerFactory remoteManagerFactory) throws Exception { System.out.println(LOG_PREFIX + name.getMethodName()); LoggingUtils.setLoggingLevel(Level.INFO); @@ -188,7 +259,7 @@ public void testRemoteManagerOrderingGuaranteeTest() throws Exception { final String hostAddress = localAddressProvider.getLocalAddress(); - final RemoteManager rm = this.remoteManagerFactory.getInstance( + final RemoteManager rm = remoteManagerFactory.getInstance( "name", hostAddress, 0, codec, new LoggingEventHandler(), true, 3, 10000, localAddressProvider, Tang.Factory.getTang().newInjector().getInstance(TcpPortProvider.class)); @@ -217,8 +288,7 @@ public void testRemoteManagerOrderingGuaranteeTest() throws Exception { timer.close(); } - @Test - public void testRemoteManagerPBufTest() throws Exception { + private void remoteManagerPBufTest(final RemoteManagerFactory remoteManagerFactory) throws Exception { System.out.println(LOG_PREFIX + name.getMethodName()); LoggingUtils.setLoggingLevel(Level.INFO); @@ -231,7 +301,7 @@ public void testRemoteManagerPBufTest() throws Exception { final String hostAddress = localAddressProvider.getLocalAddress(); - final RemoteManager rm = this.remoteManagerFactory.getInstance( + final RemoteManager rm = remoteManagerFactory.getInstance( "name", hostAddress, 0, codec, new LoggingEventHandler(), false, 3, 10000, localAddressProvider, Tang.Factory.getTang().newInjector().getInstance(TcpPortProvider.class)); @@ -253,8 +323,7 @@ public void testRemoteManagerPBufTest() throws Exception { timer.close(); } - @Test - public void testRemoteManagerExceptionTest() { + private void remoteManagerExceptionTest(final RemoteManagerFactory remoteManagerFactory) { System.out.println(LOG_PREFIX + name.getMethodName()); LoggingUtils.setLoggingLevel(Level.INFO); @@ -285,7 +354,8 @@ public void testRemoteManagerExceptionTest() { } } - private RemoteManager getTestRemoteManager(final String rmName, final int localPort, + private RemoteManager getTestRemoteManager(final RemoteManagerFactory remoteManagerFactory, + final String rmName, final int localPort, final int retry, final int retryTimeout) { final Map, Codec> clazzToCodecMap = new HashMap<>(); clazzToCodecMap.put(StartEvent.class, new ObjectSerializableCodec()); diff --git a/lang/java/reef-wake/wake/src/test/java/org/apache/reef/wake/test/remote/RemoteManagerTestHttp.java b/lang/java/reef-wake/wake/src/test/java/org/apache/reef/wake/test/remote/RemoteManagerTestHttp.java deleted file mode 100644 index f506016fec..0000000000 --- a/lang/java/reef-wake/wake/src/test/java/org/apache/reef/wake/test/remote/RemoteManagerTestHttp.java +++ /dev/null @@ -1,490 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.reef.wake.test.remote; - -import org.apache.reef.tang.Injector; -import org.apache.reef.tang.JavaConfigurationBuilder; -import org.apache.reef.tang.Tang; -import org.apache.reef.tang.exceptions.InjectionException; -import org.apache.reef.wake.EventHandler; -import org.apache.reef.wake.impl.LoggingEventHandler; -import org.apache.reef.wake.impl.LoggingUtils; -import org.apache.reef.wake.impl.TimerStage; -import org.apache.reef.wake.remote.*; -import org.apache.reef.wake.remote.address.LocalAddressProvider; -import org.apache.reef.wake.remote.impl.DefaultRemoteIdentifierFactoryImplementation; -import org.apache.reef.wake.remote.impl.MultiCodec; -import org.apache.reef.wake.remote.impl.ObjectSerializableCodec; -import org.apache.reef.wake.remote.ports.TcpPortProvider; -import org.apache.reef.wake.remote.transport.TransportFactory; -import org.apache.reef.wake.remote.transport.netty.MessagingTransportFactory; -import org.apache.reef.wake.test.util.Monitor; -import org.apache.reef.wake.test.util.TimeoutHandler; -import org.junit.Assert; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TestName; - -import java.net.UnknownHostException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.Map; -import java.util.concurrent.Callable; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.logging.Level; - -/** - * Tests for RemoteManagerFactory with HTTP Transport implementation. - */ -public class RemoteManagerTestHttp { - - private final LocalAddressProvider localAddressProvider; - private final RemoteManagerFactory remoteManagerFactory; - - public RemoteManagerTestHttp() throws InjectionException { - final JavaConfigurationBuilder builder = Tang.Factory.getTang().newConfigurationBuilder(); - builder.bindImplementation(TransportFactory.class, MessagingTransportFactory.class); - builder.bindNamedParameter(RemoteConfiguration.Protocol.class, "101"); - final Injector injector = Tang.Factory.getTang().newInjector(builder.build()); - this.localAddressProvider = injector.getInstance(LocalAddressProvider.class); - this.remoteManagerFactory = injector.getInstance(RemoteManagerFactory.class); - } - - @Rule - public final TestName name = new TestName(); - - private static final String LOG_PREFIX = "TEST "; - - @Test - public void testRemoteManagerTest() throws Exception { - System.out.println(LOG_PREFIX + name.getMethodName()); - LoggingUtils.setLoggingLevel(Level.INFO); - - final Monitor monitor = new Monitor(); - final TimerStage timer = new TimerStage(new TimeoutHandler(monitor), 2000, 2000); - - final Map, Codec> clazzToCodecMap = new HashMap<>(); - clazzToCodecMap.put(StartEvent.class, new ObjectSerializableCodec()); - clazzToCodecMap.put(TestEvent.class, new ObjectSerializableCodec()); - clazzToCodecMap.put(TestEvent1.class, new ObjectSerializableCodec()); - clazzToCodecMap.put(TestEvent2.class, new ObjectSerializableCodec()); - final Codec codec = new MultiCodec(clazzToCodecMap); - - final String hostAddress = localAddressProvider.getLocalAddress(); - - final RemoteManager rm = this.remoteManagerFactory.getInstance( - "name", hostAddress, 0, codec, new LoggingEventHandler(), false, 3, 10000, - localAddressProvider, Tang.Factory.getTang().newInjector().getInstance(TcpPortProvider.class)); - - final RemoteIdentifier remoteId = rm.getMyIdentifier(); - Assert.assertTrue(rm.getMyIdentifier().equals(remoteId)); - - final EventHandler proxyConnection = rm.getHandler(remoteId, StartEvent.class); - final EventHandler proxyHandler1 = rm.getHandler(remoteId, TestEvent1.class); - final EventHandler proxyHandler2 = rm.getHandler(remoteId, TestEvent2.class); - - final AtomicInteger counter = new AtomicInteger(0); - final int finalSize = 2; - rm.registerHandler(StartEvent.class, new MessageTypeEventHandler(rm, monitor, counter, finalSize)); - - proxyConnection.onNext(new StartEvent()); - - monitor.mwait(); - proxyHandler1.onNext(new TestEvent1("hello1", 0.0)); // registration after send expected to fail - proxyHandler2.onNext(new TestEvent2("hello2", 1.0)); - - monitor.mwait(); - - Assert.assertEquals(finalSize, counter.get()); - - rm.close(); - timer.close(); - } - - @Test - public void testRemoteManagerConnectionRetryTest() throws Exception { - final ExecutorService smExecutor = Executors.newFixedThreadPool(1); - final ExecutorService rmExecutor = Executors.newFixedThreadPool(1); - - final RemoteManager sendingManager = getTestRemoteManager("sender", 9020, 3, 2000); - - final Future smFuture = smExecutor.submit(new SendingRemoteManagerThread(sendingManager, 9010, 20000)); - Thread.sleep(1000); - - final RemoteManager receivingManager = getTestRemoteManager("receiver", 9010, 1, 2000); - final Future rmFuture = rmExecutor.submit(new ReceivingRemoteManagerThread(receivingManager, 20000, 1, 2)); - - final int smCnt = smFuture.get(); - final int rmCnt = rmFuture.get(); - - receivingManager.close(); - sendingManager.close(); - - Assert.assertEquals(0, smCnt); - Assert.assertEquals(2, rmCnt); - } - - @Test - public void testRemoteManagerConnectionRetryWithMultipleSenderTest() throws Exception { - final int numOfSenderThreads = 5; - final ExecutorService smExecutor = Executors.newFixedThreadPool(numOfSenderThreads); - final ExecutorService rmExecutor = Executors.newFixedThreadPool(1); - final ArrayList> smFutures = new ArrayList<>(numOfSenderThreads); - - final RemoteManager sendingManager = getTestRemoteManager("sender", 9030, 3, 5000); - - for (int i = 0; i < numOfSenderThreads; i++) { - smFutures.add(smExecutor.submit(new SendingRemoteManagerThread(sendingManager, 9010, 20000))); - } - - Thread.sleep(2000); - - final RemoteManager receivingManager = getTestRemoteManager("receiver", 9010, 1, 2000); - final Future receivingFuture = - rmExecutor.submit(new ReceivingRemoteManagerThread(receivingManager, 20000, numOfSenderThreads, 2)); - - // waiting sending remote manager. - for (final Future future : smFutures) { - future.get(); - } - - // waiting receiving remote manager - final int rmCnt = receivingFuture.get(); - - sendingManager.close(); - receivingManager.close(); - - // get the result - Assert.assertEquals(2 * numOfSenderThreads, rmCnt); - } - - @Test - public void testRemoteManagerOrderingGuaranteeTest() throws Exception { - System.out.println(LOG_PREFIX + name.getMethodName()); - LoggingUtils.setLoggingLevel(Level.INFO); - - final Monitor monitor = new Monitor(); - final TimerStage timer = new TimerStage(new TimeoutHandler(monitor), 2000, 2000); - - final Map, Codec> clazzToCodecMap = new HashMap<>(); - clazzToCodecMap.put(StartEvent.class, new ObjectSerializableCodec()); - clazzToCodecMap.put(TestEvent.class, new ObjectSerializableCodec()); - clazzToCodecMap.put(TestEvent1.class, new ObjectSerializableCodec()); - clazzToCodecMap.put(TestEvent2.class, new ObjectSerializableCodec()); - final Codec codec = new MultiCodec(clazzToCodecMap); - - final String hostAddress = localAddressProvider.getLocalAddress(); - - final RemoteManager rm = this.remoteManagerFactory.getInstance( - "name", hostAddress, 0, codec, new LoggingEventHandler(), true, 3, 10000, - localAddressProvider, Tang.Factory.getTang().newInjector().getInstance(TcpPortProvider.class)); - - final RemoteIdentifier remoteId = rm.getMyIdentifier(); - - final EventHandler proxyConnection = rm.getHandler(remoteId, StartEvent.class); - final EventHandler proxyHandler1 = rm.getHandler(remoteId, TestEvent1.class); - final EventHandler proxyHandler2 = rm.getHandler(remoteId, TestEvent2.class); - - final AtomicInteger counter = new AtomicInteger(0); - final int finalSize = 2; - rm.registerHandler(StartEvent.class, new MessageTypeEventHandler(rm, monitor, counter, finalSize)); - - proxyConnection.onNext(new StartEvent()); - - monitor.mwait(); - - proxyHandler1.onNext(new TestEvent1("hello1", 0.0)); - proxyHandler2.onNext(new TestEvent2("hello2", 1.0)); - - monitor.mwait(); - - Assert.assertEquals(finalSize, counter.get()); - - rm.close(); - timer.close(); - } - - @Test - public void testRemoteManagerPBufTest() throws Exception { - System.out.println(LOG_PREFIX + name.getMethodName()); - LoggingUtils.setLoggingLevel(Level.INFO); - - final Monitor monitor = new Monitor(); - final TimerStage timer = new TimerStage(new TimeoutHandler(monitor), 2000, 2000); - - final Map, Codec> clazzToCodecMap = new HashMap<>(); - clazzToCodecMap.put(TestEvent.class, new TestEventCodec()); - final Codec codec = new MultiCodec(clazzToCodecMap); - - final String hostAddress = localAddressProvider.getLocalAddress(); - - final RemoteManager rm = this.remoteManagerFactory.getInstance( - "name", hostAddress, 0, codec, new LoggingEventHandler(), false, 3, 10000, - localAddressProvider, Tang.Factory.getTang().newInjector().getInstance(TcpPortProvider.class)); - - final RemoteIdentifier remoteId = rm.getMyIdentifier(); - - final EventHandler proxyHandler = rm.getHandler(remoteId, TestEvent.class); - - final AtomicInteger counter = new AtomicInteger(0); - final int finalSize = 0; - rm.registerHandler(TestEvent.class, new MessageTypeEventHandler(rm, monitor, counter, finalSize)); - - proxyHandler.onNext(new TestEvent("hello", 0.0)); - - monitor.mwait(); - - Assert.assertEquals(finalSize, counter.get()); - - rm.close(); - timer.close(); - } - - @Test - public void testRemoteManagerExceptionTest() { - System.out.println(LOG_PREFIX + name.getMethodName()); - LoggingUtils.setLoggingLevel(Level.INFO); - - final Monitor monitor = new Monitor(); - final TimerStage timer = new TimerStage(new TimeoutHandler(monitor), 2000, 2000); - - final Map, Codec> clazzToCodecMap = new HashMap<>(); - clazzToCodecMap.put(StartEvent.class, new ObjectSerializableCodec()); - clazzToCodecMap.put(TestEvent.class, new ObjectSerializableCodec()); - final Codec codec = new MultiCodec(clazzToCodecMap); - - final ExceptionHandler errorHandler = new ExceptionHandler(monitor); - - try (final RemoteManager rm = remoteManagerFactory.getInstance("name", 0, codec, errorHandler)) { - final RemoteIdentifier remoteId = rm.getMyIdentifier(); - - final EventHandler proxyConnection = rm.getHandler(remoteId, StartEvent.class); - rm.registerHandler(StartEvent.class, new ExceptionGenEventHandler("recvExceptionGen")); - - proxyConnection.onNext(new StartEvent()); - monitor.mwait(); - timer.close(); - - } catch (final UnknownHostException e) { - e.printStackTrace(); - } catch (final Exception e) { - e.printStackTrace(); - } - } - - private RemoteManager getTestRemoteManager(final String rmName, final int localPort, - final int retry, final int retryTimeout) { - final Map, Codec> clazzToCodecMap = new HashMap<>(); - clazzToCodecMap.put(StartEvent.class, new ObjectSerializableCodec()); - clazzToCodecMap.put(TestEvent1.class, new ObjectSerializableCodec()); - clazzToCodecMap.put(TestEvent2.class, new ObjectSerializableCodec()); - final Codec codec = new MultiCodec(clazzToCodecMap); - - final String hostAddress = localAddressProvider.getLocalAddress(); - try { - TcpPortProvider tcpPortProvider = Tang.Factory.getTang().newInjector().getInstance(TcpPortProvider.class); - return remoteManagerFactory.getInstance(rmName, hostAddress, localPort, - codec, new LoggingEventHandler(), false, retry, retryTimeout, - localAddressProvider, tcpPortProvider); - } catch (final InjectionException e) { - throw new RuntimeException(e); - } - } - - private class SendingRemoteManagerThread implements Callable { - - private final int remotePort; - private final int timeout; - private RemoteManager rm; - - SendingRemoteManagerThread(final RemoteManager rm, final int remotePort, final int timeout) { - this.remotePort = remotePort; - this.timeout = timeout; - this.rm = rm; - } - - @Override - public Integer call() throws Exception { - - final Monitor monitor = new Monitor(); - final TimerStage timer = new TimerStage(new TimeoutHandler(monitor), timeout, timeout); - - final String hostAddress = localAddressProvider.getLocalAddress(); - final RemoteIdentifierFactory factory = new DefaultRemoteIdentifierFactoryImplementation(); - final RemoteIdentifier remoteId = factory.getNewInstance("socket://" + hostAddress + ":" + remotePort); - - final EventHandler proxyConnection = rm.getHandler(remoteId, StartEvent.class); - final EventHandler proxyHandler1 = rm.getHandler(remoteId, TestEvent1.class); - final EventHandler proxyHandler2 = rm.getHandler(remoteId, TestEvent2.class); - - final AtomicInteger counter = new AtomicInteger(0); - final int finalSize = 0; - rm.registerHandler(StartEvent.class, new MessageTypeEventHandler(rm, monitor, counter, finalSize)); - - proxyConnection.onNext(new StartEvent()); - - monitor.mwait(); - - proxyHandler1.onNext(new TestEvent1("hello1", 0.0)); // registration after send expected to fail - proxyHandler2.onNext(new TestEvent2("hello2", 0.0)); // registration after send expected to fail - timer.close(); - - return counter.get(); - } - } - - private class ReceivingRemoteManagerThread implements Callable { - - private final int timeout; - private final int numOfConnection; - private final int numOfEvent; - private RemoteManager rm; - - ReceivingRemoteManagerThread(final RemoteManager rm, final int timeout, - final int numOfConnection, final int numOfEvent) { - this.rm = rm; - this.timeout = timeout; - this.numOfConnection = numOfConnection; - this.numOfEvent = numOfEvent; - } - - @Override - public Integer call() throws Exception { - - final Monitor monitor = new Monitor(); - final TimerStage timer = new TimerStage(new TimeoutHandler(monitor), timeout, timeout); - - final AtomicInteger counter = new AtomicInteger(0); - final int finalSize = numOfConnection * numOfEvent; - rm.registerHandler(StartEvent.class, new MessageTypeEventHandler(rm, monitor, counter, finalSize)); - - for (int i = 0; i < numOfConnection; i++) { - monitor.mwait(); - } - monitor.mwait(); - timer.close(); - return counter.get(); - } - } - - class MessageTypeEventHandler implements EventHandler> { - - private final RemoteManager rm; - private final Monitor monitor; - private final AtomicInteger counter; - private final int finalSize; - - MessageTypeEventHandler(final RemoteManager rm, final Monitor monitor, - final AtomicInteger counter, final int finalSize) { - this.rm = rm; - this.monitor = monitor; - this.counter = counter; - this.finalSize = finalSize; - } - - @Override - public void onNext(final RemoteMessage value) { - - final RemoteIdentifier id = value.getIdentifier(); - final T message = value.getMessage(); - - System.out.println(this.getClass() + " " + value + " " + id.toString() + " " + message.toString()); - - System.out.println("Sleeping to force a bug"); - // try { - // Thread.sleep(2000); - //} catch (InterruptedException e) { - - // e.printStackTrace(); - // } - - // register specific handlers - rm.registerHandler(id, TestEvent1.class, - new ConsoleEventHandler("console1", monitor, counter, finalSize)); - rm.registerHandler(id, TestEvent2.class, - new ConsoleEventHandler("console2", monitor, counter, finalSize)); - monitor.mnotify(); - } - } - - class ConsoleEventHandler implements EventHandler { - - private final String name; - private final Monitor monitor; - private final AtomicInteger counter; - private final int finalSize; - - ConsoleEventHandler(final String name, final Monitor monitor, final AtomicInteger counter, final int finalSize) { - this.name = name; - this.monitor = monitor; - this.counter = counter; - this.finalSize = finalSize; - } - - @Override - public void onNext(final T value) { - System.out.println(this.getClass() + " " + name + " " + value); - if (counter.incrementAndGet() == finalSize) { - System.out.println(this.getClass() + " notify counter: " + counter.get()); - monitor.mnotify(); - } - } - } - - class ExceptionGenEventHandler implements EventHandler> { - - private final String name; - - ExceptionGenEventHandler(final String name) { - this.name = name; - } - - @Override - public void onNext(final RemoteMessage value) { - System.out.println(name + " " + value); - throw new TestRuntimeException("Test exception"); - } - } - - class ExceptionHandler implements EventHandler { - - private final Monitor monitor; - - ExceptionHandler(final Monitor monitor) { - this.monitor = monitor; - } - - @Override - public void onNext(final Throwable value) { - System.out.println("!!! ExceptionHandler called : " + value); - monitor.mnotify(); - } - } - - final class TestRuntimeException extends RuntimeException { - private static final long serialVersionUID = 1L; - - TestRuntimeException(final String s) { - super(s); - } - } -} From 017154e8087a95f34d49b1f992ba4d581df92c7a Mon Sep 17 00:00:00 2001 From: Kim Doyoung Date: Thu, 23 Nov 2017 17:31:46 +0900 Subject: [PATCH 24/34] changed implementation of ChannelInitializer, Http Event Listener --- .../netty/NettyChannelInitializer.java | 22 +++--- .../netty/NettyHttpClientEventListener.java | 32 ++++---- .../netty/NettyHttpServerEventListener.java | 73 +++++++++---------- 3 files changed, 59 insertions(+), 68 deletions(-) diff --git a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyChannelInitializer.java b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyChannelInitializer.java index 59a309a4f5..2cf860eb3a 100644 --- a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyChannelInitializer.java +++ b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyChannelInitializer.java @@ -19,6 +19,7 @@ package org.apache.reef.wake.remote.transport.netty; import io.netty.channel.ChannelInitializer; +import io.netty.channel.ChannelPipeline; import io.netty.channel.socket.SocketChannel; import io.netty.handler.codec.LengthFieldBasedFrameDecoder; import io.netty.handler.codec.LengthFieldPrepender; @@ -37,6 +38,7 @@ class NettyChannelInitializer extends ChannelInitializer { * the buffer size of the frame decoder. */ public static final int MAXFRAMELENGTH = 10 * 1024 * 1024; + private static final int MAX_HTTP_MESSAGE_LENGTH = 10 * 1024 * 1024; private final NettyChannelHandlerFactory handlerFactory; /** @@ -62,32 +64,32 @@ class NettyChannelInitializer extends ChannelInitializer { @Override protected void initChannel(final SocketChannel ch) throws Exception { + final ChannelPipeline pipeline = ch.pipeline(); switch (this.protocolType) { case TCP: - ch.pipeline() + pipeline .addLast("frameDecoder", new LengthFieldBasedFrameDecoder(MAXFRAMELENGTH, 0, 4, 0, 4)) .addLast("bytesDecoder", new ByteArrayDecoder()) .addLast("frameEncoder", new LengthFieldPrepender(4)) .addLast("bytesEncoder", new ByteArrayEncoder()) - .addLast("chunker", new ChunkedReadWriteHandler()) - .addLast("handler", handlerFactory.createChannelInboundHandler()); + .addLast("chunker", new ChunkedReadWriteHandler()); break; case HTTP: if(isServer) { - ch.pipeline() + pipeline .addLast("codec", new HttpServerCodec()) .addLast("requestDecoder", new HttpRequestDecoder()) - .addLast("responseEncoder", new HttpResponseEncoder()) - .addLast("handler", handlerFactory.createChannelInboundHandler()); + .addLast("responseEncoder", new HttpResponseEncoder()); } else { - ch.pipeline() - .addLast("codec", new HttpClientCodec()) - .addLast("decompressor", new HttpContentDecompressor()) - .addLast("handler", handlerFactory.createChannelInboundHandler()); + pipeline + .addLast("codec", new HttpClientCodec()); } + pipeline.addLast("aggregator", new HttpObjectAggregator(MAX_HTTP_MESSAGE_LENGTH)); break; default: throw new IllegalArgumentException("Invalid type of channel"); } + // every channel's pipeline have a same inbound handler. + pipeline.addLast("handler", handlerFactory.createChannelInboundHandler()); } } diff --git a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyHttpClientEventListener.java b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyHttpClientEventListener.java index ddc55f9477..b5d720fe39 100644 --- a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyHttpClientEventListener.java +++ b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyHttpClientEventListener.java @@ -21,9 +21,8 @@ import io.netty.buffer.ByteBuf; import io.netty.channel.Channel; import io.netty.channel.ChannelHandlerContext; +import io.netty.handler.codec.http.FullHttpResponse; import io.netty.handler.codec.http.HttpContent; -import io.netty.handler.codec.http.HttpResponse; -import io.netty.handler.codec.http.LastHttpContent; import io.netty.util.CharsetUtil; import org.apache.reef.wake.EStage; import org.apache.reef.wake.remote.impl.TransportEvent; @@ -49,31 +48,26 @@ final class NettyHttpClientEventListener extends AbstractNettyEventListener { @Override public void channelRead(final ChannelHandlerContext ctx, final Object msg) { - if (msg instanceof HttpResponse) { + if (msg instanceof FullHttpResponse) { if(LOG.isLoggable(Level.FINEST)) { LOG.log(Level.FINEST, "HttpResponse Received: {0}", msg); } - } else if (msg instanceof HttpContent) { final HttpContent httpContent = (HttpContent) msg; final ByteBuf content = httpContent.content(); - if (LOG.isLoggable(Level.FINEST) && content.isReadable()) { + final Channel channel = ctx.channel(); + final byte[] message = new byte[content.readableBytes()]; + + content.readBytes(message); + if (LOG.isLoggable(Level.FINEST)) { buf.append("CONTENT: ").append(content.toString(CharsetUtil.UTF_8)).append("\r\n"); + LOG.log(Level.FINEST, "MessageEvent: local: {0} remote: {1} :: {2}", new Object[]{ + channel.localAddress(), channel.remoteAddress(), buf}); + buf.setLength(0); // clearing the buffer } - if (msg instanceof LastHttpContent) { - final Channel channel = ctx.channel(); - final byte[] message = new byte[content.readableBytes()]; - content.readBytes(message); - if (LOG.isLoggable(Level.FINEST)) { - LOG.log(Level.FINEST, "MessageEvent: local: {0} remote: {1} :: {2}", new Object[]{ - channel.localAddress(), channel.remoteAddress(), buf}); - buf.setLength(0); // clearing the buffer - } - - if (message.length > 0) { - // send to the dispatch stage - this.stage.onNext(this.getTransportEvent(message, channel)); - } + if (message.length > 0) { + // send to the dispatch stage + this.stage.onNext(this.getTransportEvent(message, channel)); } } else { LOG.log(Level.SEVERE, "Unknown type of message received: {0}", msg); diff --git a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyHttpServerEventListener.java b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyHttpServerEventListener.java index 70618e2de0..ab880e77ed 100644 --- a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyHttpServerEventListener.java +++ b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyHttpServerEventListener.java @@ -39,7 +39,6 @@ */ final class NettyHttpServerEventListener extends AbstractNettyEventListener { - private HttpRequest httpRequest; private final StringBuilder buf = new StringBuilder(); private final URI uri; private final NettyLinkFactory linkFactory; @@ -72,40 +71,36 @@ public void channelActive(final ChannelHandlerContext ctx) { @Override public void channelRead(final ChannelHandlerContext ctx, final Object msg) { - if(msg instanceof HttpRequest) { + if(msg instanceof FullHttpRequest) { LOG.log(Level.FINEST, "HttpRequest received"); - final HttpRequest request = (HttpRequest) msg; + final FullHttpRequest request = (FullHttpRequest) msg; final HttpHeaders headers = request.headers(); - this.httpRequest = request; - - if (!headers.isEmpty()) { - for (final Map.Entry h : headers) { - final CharSequence key = h.getKey(); - final CharSequence value = h.getValue(); - buf.append("HEADER: ").append(key).append(" = ").append(value).append("\r\n"); + final ByteBuf content = request.content(); + + if (LOG.isLoggable(Level.FINEST)) { + // log header to trailing header contents. + if (!headers.isEmpty()) { + for (final Map.Entry h : headers) { + final CharSequence key = h.getKey(); + final CharSequence value = h.getValue(); + buf.append("HEADER: ").append(key).append(" = ").append(value).append("\r\n"); + } + buf.append("\r\n"); + } + appendDecoderResult(buf, request); + if (content.isReadable()) { + buf.append("CONTENT: "); + buf.append(content.toString(CharsetUtil.UTF_8)); + buf.append("\r\n"); + appendDecoderResult(buf, request); } - buf.append("\r\n"); - } - appendDecoderResult(buf, request); - } else if (msg instanceof HttpContent) { - LOG.log(Level.FINEST, "HttpContent received"); - final HttpContent httpContent = (HttpContent) msg; - final ByteBuf content = httpContent.content(); - if (content.isReadable()) { - buf.append("CONTENT: "); - buf.append(content.toString(CharsetUtil.UTF_8)); - buf.append("\r\n"); - appendDecoderResult(buf, this.httpRequest); - } - if (msg instanceof LastHttpContent) { buf.append("END OF CONTENT\r\n"); - final LastHttpContent trailer = (LastHttpContent) msg; - if (!trailer.trailingHeaders().isEmpty()) { + if (!request.trailingHeaders().isEmpty()) { buf.append("\r\n"); - for (CharSequence name: trailer.trailingHeaders().names()) { - for (CharSequence value: trailer.trailingHeaders().getAll(name)) { + for (CharSequence name : request.trailingHeaders().names()) { + for (CharSequence value : request.trailingHeaders().getAll(name)) { buf.append("TRAILING HEADER: "); buf.append(name).append(" = ").append(value).append("\r\n"); } @@ -113,19 +108,19 @@ public void channelRead(final ChannelHandlerContext ctx, final Object msg) { buf.append("\r\n"); buf.setLength(0); // clearing the buffer } + } - final Channel channel = ctx.channel(); - final byte[] message = new byte[content.readableBytes()]; - content.readBytes(message); - if (LOG.isLoggable(Level.FINEST)) { - LOG.log(Level.FINEST, "MessageEvent: local: {0} remote: {1} :: {2}", new Object[]{ - channel.localAddress(), channel.remoteAddress(), content}); - } + final Channel channel = ctx.channel(); + final byte[] message = new byte[content.readableBytes()]; + content.readBytes(message); + if (LOG.isLoggable(Level.FINEST)) { + LOG.log(Level.FINEST, "MessageEvent: local: {0} remote: {1} :: {2}", new Object[]{ + channel.localAddress(), channel.remoteAddress(), content}); + } - if (message.length > 0) { - // send to the dispatch stage - this.stage.onNext(this.getTransportEvent(message, channel)); - } + if (message.length > 0) { + // send to the dispatch stage + this.stage.onNext(this.getTransportEvent(message, channel)); } } else { LOG.log(Level.SEVERE, "Unknown type of message received: {0}", msg); From f221dbb13464465435f299ae212622c818877dd1 Mon Sep 17 00:00:00 2001 From: Kim Doyoung Date: Fri, 24 Nov 2017 15:53:44 +0900 Subject: [PATCH 25/34] fixed NettyChannelInitializer and Event Listener, NettyHttpLink --- .../transport/netty/NettyChannelInitializer.java | 15 ++++----------- .../netty/NettyHttpClientEventListener.java | 5 +++-- .../remote/transport/netty/NettyHttpLink.java | 2 +- .../netty/NettyHttpServerEventListener.java | 7 +++---- 4 files changed, 11 insertions(+), 18 deletions(-) diff --git a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyChannelInitializer.java b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyChannelInitializer.java index 2cf860eb3a..0de7b082d1 100644 --- a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyChannelInitializer.java +++ b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyChannelInitializer.java @@ -75,21 +75,14 @@ protected void initChannel(final SocketChannel ch) throws Exception { .addLast("chunker", new ChunkedReadWriteHandler()); break; case HTTP: - if(isServer) { - pipeline - .addLast("codec", new HttpServerCodec()) - .addLast("requestDecoder", new HttpRequestDecoder()) - .addLast("responseEncoder", new HttpResponseEncoder()); - } else { - pipeline - .addLast("codec", new HttpClientCodec()); - } - pipeline.addLast("aggregator", new HttpObjectAggregator(MAX_HTTP_MESSAGE_LENGTH)); + pipeline + .addLast("codec", isServer ? new HttpServerCodec() : new HttpClientCodec()) + .addLast("aggregator", new HttpObjectAggregator(MAX_HTTP_MESSAGE_LENGTH)); break; default: throw new IllegalArgumentException("Invalid type of channel"); } - // every channel's pipeline have a same inbound handler. + // every channel pipeline has the same inbound handler. pipeline.addLast("handler", handlerFactory.createChannelInboundHandler()); } } diff --git a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyHttpClientEventListener.java b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyHttpClientEventListener.java index b5d720fe39..6c3202ddad 100644 --- a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyHttpClientEventListener.java +++ b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyHttpClientEventListener.java @@ -25,6 +25,7 @@ import io.netty.handler.codec.http.HttpContent; import io.netty.util.CharsetUtil; import org.apache.reef.wake.EStage; +import org.apache.reef.wake.remote.exception.RemoteRuntimeException; import org.apache.reef.wake.remote.impl.TransportEvent; import java.net.SocketAddress; @@ -38,7 +39,6 @@ final class NettyHttpClientEventListener extends AbstractNettyEventListener { private static final Logger LOG = Logger.getLogger(NettyHttpClientEventListener.class.getName()); - private final StringBuilder buf = new StringBuilder(); NettyHttpClientEventListener( final ConcurrentMap addrToLinkRefMap, @@ -56,6 +56,7 @@ public void channelRead(final ChannelHandlerContext ctx, final Object msg) { final ByteBuf content = httpContent.content(); final Channel channel = ctx.channel(); final byte[] message = new byte[content.readableBytes()]; + final StringBuilder buf = new StringBuilder(); content.readBytes(message); if (LOG.isLoggable(Level.FINEST)) { @@ -71,8 +72,8 @@ public void channelRead(final ChannelHandlerContext ctx, final Object msg) { } } else { LOG.log(Level.SEVERE, "Unknown type of message received: {0}", msg); + throw new RemoteRuntimeException("Unknown type of message received " + msg); } - } @Override diff --git a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyHttpLink.java b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyHttpLink.java index 2dc38e78af..315009aa9b 100644 --- a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyHttpLink.java +++ b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyHttpLink.java @@ -100,10 +100,10 @@ public void write(final T message) { .set(HttpHeaders.Names.CONTENT_LENGTH, buf.readableBytes()); request.content().clear().writeBytes(buf); final ChannelFuture future = channel.writeAndFlush(request); - future.sync(); if (listener != null) { future.addListener(new NettyChannelFutureListener<>(message, listener)); } + future.sync(); } catch (final InterruptedException ex) { LOG.log(Level.SEVERE, "Cannot send request to " + uri.getHost(), ex); } diff --git a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyHttpServerEventListener.java b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyHttpServerEventListener.java index ab880e77ed..8d3819e0db 100644 --- a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyHttpServerEventListener.java +++ b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyHttpServerEventListener.java @@ -25,6 +25,7 @@ import io.netty.handler.codec.http.*; import io.netty.util.CharsetUtil; import org.apache.reef.wake.EStage; +import org.apache.reef.wake.remote.exception.RemoteRuntimeException; import org.apache.reef.wake.remote.impl.ByteCodec; import org.apache.reef.wake.remote.impl.TransportEvent; @@ -39,7 +40,6 @@ */ final class NettyHttpServerEventListener extends AbstractNettyEventListener { - private final StringBuilder buf = new StringBuilder(); private final URI uri; private final NettyLinkFactory linkFactory; @@ -77,6 +77,7 @@ public void channelRead(final ChannelHandlerContext ctx, final Object msg) { final FullHttpRequest request = (FullHttpRequest) msg; final HttpHeaders headers = request.headers(); final ByteBuf content = request.content(); + final StringBuilder buf = new StringBuilder(); if (LOG.isLoggable(Level.FINEST)) { // log header to trailing header contents. @@ -124,9 +125,7 @@ public void channelRead(final ChannelHandlerContext ctx, final Object msg) { } } else { LOG.log(Level.SEVERE, "Unknown type of message received: {0}", msg); - } - if(LOG.isLoggable(Level.FINEST)) { - LOG.log(Level.FINEST, "Message received {0}", buf); + throw new RemoteRuntimeException("Unknown type of message received " + msg); } } From 455464a988d51b76b85bf7cc46e70f639b30b56d Mon Sep 17 00:00:00 2001 From: Kim Doyoung Date: Tue, 5 Dec 2017 16:58:30 +0900 Subject: [PATCH 26/34] changed implementation in Netty[HttpServer,Client]EventListener logging buffer's behavior has changed, content variable also changed --- .../netty/NettyHttpClientEventListener.java | 21 +++++++----- .../netty/NettyHttpServerEventListener.java | 33 ++++++++++--------- 2 files changed, 30 insertions(+), 24 deletions(-) diff --git a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyHttpClientEventListener.java b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyHttpClientEventListener.java index 6c3202ddad..814e423ff1 100644 --- a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyHttpClientEventListener.java +++ b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyHttpClientEventListener.java @@ -53,22 +53,27 @@ public void channelRead(final ChannelHandlerContext ctx, final Object msg) { LOG.log(Level.FINEST, "HttpResponse Received: {0}", msg); } final HttpContent httpContent = (HttpContent) msg; - final ByteBuf content = httpContent.content(); + final ByteBuf byteBuf = httpContent.content(); final Channel channel = ctx.channel(); - final byte[] message = new byte[content.readableBytes()]; - final StringBuilder buf = new StringBuilder(); + final byte[] content; + + if (byteBuf.hasArray()) { + content = byteBuf.array(); + } else { + content = new byte[byteBuf.readableBytes()]; + byteBuf.readBytes(content); + } - content.readBytes(message); if (LOG.isLoggable(Level.FINEST)) { - buf.append("CONTENT: ").append(content.toString(CharsetUtil.UTF_8)).append("\r\n"); + final StringBuilder buf = new StringBuilder(); + buf.append("CONTENT: ").append(byteBuf.toString(CharsetUtil.UTF_8)).append("\r\n"); LOG.log(Level.FINEST, "MessageEvent: local: {0} remote: {1} :: {2}", new Object[]{ channel.localAddress(), channel.remoteAddress(), buf}); - buf.setLength(0); // clearing the buffer } - if (message.length > 0) { + if (content.length > 0) { // send to the dispatch stage - this.stage.onNext(this.getTransportEvent(message, channel)); + this.stage.onNext(this.getTransportEvent(content, channel)); } } else { LOG.log(Level.SEVERE, "Unknown type of message received: {0}", msg); diff --git a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyHttpServerEventListener.java b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyHttpServerEventListener.java index 8d3819e0db..78d4881a6a 100644 --- a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyHttpServerEventListener.java +++ b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyHttpServerEventListener.java @@ -72,15 +72,22 @@ public void channelActive(final ChannelHandlerContext ctx) { @Override public void channelRead(final ChannelHandlerContext ctx, final Object msg) { if(msg instanceof FullHttpRequest) { - LOG.log(Level.FINEST, "HttpRequest received"); - final FullHttpRequest request = (FullHttpRequest) msg; final HttpHeaders headers = request.headers(); - final ByteBuf content = request.content(); - final StringBuilder buf = new StringBuilder(); + final ByteBuf byteBuf = request.content(); + final Channel channel = ctx.channel(); + final byte[] content; + + if (byteBuf.hasArray()) { + content = byteBuf.array(); + } else { + content = new byte[byteBuf.readableBytes()]; + byteBuf.readBytes(content); + } if (LOG.isLoggable(Level.FINEST)) { // log header to trailing header contents. + final StringBuilder buf = new StringBuilder(); if (!headers.isEmpty()) { for (final Map.Entry h : headers) { final CharSequence key = h.getKey(); @@ -90,9 +97,9 @@ public void channelRead(final ChannelHandlerContext ctx, final Object msg) { buf.append("\r\n"); } appendDecoderResult(buf, request); - if (content.isReadable()) { + if (byteBuf.isReadable()) { buf.append("CONTENT: "); - buf.append(content.toString(CharsetUtil.UTF_8)); + buf.append(byteBuf.toString(CharsetUtil.UTF_8)); buf.append("\r\n"); appendDecoderResult(buf, request); } @@ -107,21 +114,15 @@ public void channelRead(final ChannelHandlerContext ctx, final Object msg) { } } buf.append("\r\n"); - buf.setLength(0); // clearing the buffer } - } - - final Channel channel = ctx.channel(); - final byte[] message = new byte[content.readableBytes()]; - content.readBytes(message); - if (LOG.isLoggable(Level.FINEST)) { + LOG.log(Level.FINEST, "Received Message:\n{0}", buf.toString()); LOG.log(Level.FINEST, "MessageEvent: local: {0} remote: {1} :: {2}", new Object[]{ - channel.localAddress(), channel.remoteAddress(), content}); + channel.localAddress(), channel.remoteAddress(), byteBuf}); } - if (message.length > 0) { + if (content.length > 0) { // send to the dispatch stage - this.stage.onNext(this.getTransportEvent(message, channel)); + this.stage.onNext(this.getTransportEvent(content, channel)); } } else { LOG.log(Level.SEVERE, "Unknown type of message received: {0}", msg); From bd3b205a5171c0c402351a419ea29a8743554ad7 Mon Sep 17 00:00:00 2001 From: Kim Doyoung Date: Wed, 6 Dec 2017 14:32:48 +0900 Subject: [PATCH 27/34] removed check on lenth of content --- .../transport/netty/NettyHttpClientEventListener.java | 6 ++---- .../transport/netty/NettyHttpServerEventListener.java | 6 ++---- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyHttpClientEventListener.java b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyHttpClientEventListener.java index 814e423ff1..df72cde6ab 100644 --- a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyHttpClientEventListener.java +++ b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyHttpClientEventListener.java @@ -71,10 +71,8 @@ public void channelRead(final ChannelHandlerContext ctx, final Object msg) { channel.localAddress(), channel.remoteAddress(), buf}); } - if (content.length > 0) { - // send to the dispatch stage - this.stage.onNext(this.getTransportEvent(content, channel)); - } + // send to the dispatch stage + this.stage.onNext(this.getTransportEvent(content, channel)); } else { LOG.log(Level.SEVERE, "Unknown type of message received: {0}", msg); throw new RemoteRuntimeException("Unknown type of message received " + msg); diff --git a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyHttpServerEventListener.java b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyHttpServerEventListener.java index 78d4881a6a..abc539b43b 100644 --- a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyHttpServerEventListener.java +++ b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyHttpServerEventListener.java @@ -120,10 +120,8 @@ public void channelRead(final ChannelHandlerContext ctx, final Object msg) { channel.localAddress(), channel.remoteAddress(), byteBuf}); } - if (content.length > 0) { - // send to the dispatch stage - this.stage.onNext(this.getTransportEvent(content, channel)); - } + // send to the dispatch stage + this.stage.onNext(this.getTransportEvent(content, channel)); } else { LOG.log(Level.SEVERE, "Unknown type of message received: {0}", msg); throw new RemoteRuntimeException("Unknown type of message received " + msg); From 222567996ac25cd7a413899317c7c10914f1b433 Mon Sep 17 00:00:00 2001 From: Kim Doyoung Date: Thu, 7 Dec 2017 23:05:38 +0900 Subject: [PATCH 28/34] copiedBuffer to wrappedBuffer --- .../apache/reef/wake/remote/transport/netty/NettyHttpLink.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyHttpLink.java b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyHttpLink.java index 315009aa9b..64e7fb8f4e 100644 --- a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyHttpLink.java +++ b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyHttpLink.java @@ -91,7 +91,7 @@ public void write(final T message) { try { final FullHttpRequest request = new DefaultFullHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.POST, uri.getRawPath()); - final ByteBuf buf = Unpooled.copiedBuffer(encoder.encode(message)); + final ByteBuf buf = Unpooled.wrappedBuffer(encoder.encode(message)); request.headers() .set(HttpHeaders.Names.HOST, uri.getHost()) .set(HttpHeaders.Names.CONNECTION, HttpHeaders.Values.KEEP_ALIVE) From b20df37f050a5da786373f053c3c54896b25bc22 Mon Sep 17 00:00:00 2001 From: Kim Doyoung Date: Thu, 4 Jan 2018 14:13:24 +0900 Subject: [PATCH 29/34] removed sync() in NettyHttpLink --- .../apache/reef/wake/remote/transport/netty/NettyHttpLink.java | 1 - 1 file changed, 1 deletion(-) diff --git a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyHttpLink.java b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyHttpLink.java index 64e7fb8f4e..a2ab8f7df3 100644 --- a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyHttpLink.java +++ b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyHttpLink.java @@ -103,7 +103,6 @@ public void write(final T message) { if (listener != null) { future.addListener(new NettyChannelFutureListener<>(message, listener)); } - future.sync(); } catch (final InterruptedException ex) { LOG.log(Level.SEVERE, "Cannot send request to " + uri.getHost(), ex); } From 94fb7bc9baf1198f603760889501b9e33fe5879e Mon Sep 17 00:00:00 2001 From: Kim Doyoung Date: Thu, 4 Jan 2018 14:17:44 +0900 Subject: [PATCH 30/34] removed `sync()` in NettyHttpLink --- .../remote/transport/netty/NettyHttpLink.java | 30 ++++++++----------- 1 file changed, 13 insertions(+), 17 deletions(-) diff --git a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyHttpLink.java b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyHttpLink.java index a2ab8f7df3..7fa586b195 100644 --- a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyHttpLink.java +++ b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyHttpLink.java @@ -88,23 +88,19 @@ public NettyHttpLink( @Override public void write(final T message) { LOG.log(Level.FINEST, "write {0} :: {1}", new Object[] {channel, message}); - try { - final FullHttpRequest request = - new DefaultFullHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.POST, uri.getRawPath()); - final ByteBuf buf = Unpooled.wrappedBuffer(encoder.encode(message)); - request.headers() - .set(HttpHeaders.Names.HOST, uri.getHost()) - .set(HttpHeaders.Names.CONNECTION, HttpHeaders.Values.KEEP_ALIVE) - .set(HttpHeaders.Names.ACCEPT_ENCODING, HttpHeaders.Values.GZIP) - .set(HttpHeaders.Names.CONTENT_TYPE, "application/wake-transport") - .set(HttpHeaders.Names.CONTENT_LENGTH, buf.readableBytes()); - request.content().clear().writeBytes(buf); - final ChannelFuture future = channel.writeAndFlush(request); - if (listener != null) { - future.addListener(new NettyChannelFutureListener<>(message, listener)); - } - } catch (final InterruptedException ex) { - LOG.log(Level.SEVERE, "Cannot send request to " + uri.getHost(), ex); + final FullHttpRequest request = + new DefaultFullHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.POST, uri.getRawPath()); + final ByteBuf buf = Unpooled.wrappedBuffer(encoder.encode(message)); + request.headers() + .set(HttpHeaders.Names.HOST, uri.getHost()) + .set(HttpHeaders.Names.CONNECTION, HttpHeaders.Values.KEEP_ALIVE) + .set(HttpHeaders.Names.ACCEPT_ENCODING, HttpHeaders.Values.GZIP) + .set(HttpHeaders.Names.CONTENT_TYPE, "application/wake-transport") + .set(HttpHeaders.Names.CONTENT_LENGTH, buf.readableBytes()); + request.content().clear().writeBytes(buf); + final ChannelFuture future = channel.writeAndFlush(request); + if (listener != null) { + future.addListener(new NettyChannelFutureListener<>(message, listener)); } } From 046739d3ae9641ae76f18029777543a82b5f5efc Mon Sep 17 00:00:00 2001 From: Kim Doyoung Date: Fri, 5 Jan 2018 14:12:59 +0900 Subject: [PATCH 31/34] check isLoggable --- .../reef/wake/remote/transport/netty/NettyHttpLink.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyHttpLink.java b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyHttpLink.java index 7fa586b195..b96d770975 100644 --- a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyHttpLink.java +++ b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyHttpLink.java @@ -87,7 +87,9 @@ public NettyHttpLink( */ @Override public void write(final T message) { - LOG.log(Level.FINEST, "write {0} :: {1}", new Object[] {channel, message}); + if (LOG.isLoggable(Level.FINEST)) { + LOG.log(Level.FINEST, "write {0} :: {1}", new Object[] {channel, message}); + } final FullHttpRequest request = new DefaultFullHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.POST, uri.getRawPath()); final ByteBuf buf = Unpooled.wrappedBuffer(encoder.encode(message)); From 645b8986f76f49aada9ef3cc90b955e53f92de21 Mon Sep 17 00:00:00 2001 From: Kim Doyoung Date: Mon, 12 Feb 2018 00:34:25 +0900 Subject: [PATCH 32/34] Fixed bug on uri creation --- .../remote/transport/netty/NettyMessagingTransport.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyMessagingTransport.java b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyMessagingTransport.java index 537bf8bd2b..1e0e01da3d 100644 --- a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyMessagingTransport.java +++ b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyMessagingTransport.java @@ -136,7 +136,11 @@ private NettyMessagingTransport( // for HTTP and default Netty if (protocolType == ProtocolType.HTTP) { - this.uri = URI.create("http://" + hostAddress); + try{ + this.uri = URI.create("http://" + host); + } catch (IllegalArgumentException e){ + throw new RemoteRuntimeException("Invalid host address: " + host); + } } else { this.uri = null; } From bd2030a3a98378c55e7219a21b1049a1ea92f125 Mon Sep 17 00:00:00 2001 From: Kim Doyoung Date: Thu, 22 Feb 2018 19:39:41 +0900 Subject: [PATCH 33/34] Added Exception e into throwing exception --- .../wake/remote/transport/netty/NettyMessagingTransport.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyMessagingTransport.java b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyMessagingTransport.java index 1e0e01da3d..a2a540f6d2 100644 --- a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyMessagingTransport.java +++ b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyMessagingTransport.java @@ -139,7 +139,7 @@ private NettyMessagingTransport( try{ this.uri = URI.create("http://" + host); } catch (IllegalArgumentException e){ - throw new RemoteRuntimeException("Invalid host address: " + host); + throw new RemoteRuntimeException("Invalid host address: " + host, e); } } else { this.uri = null; From 6541d3c94e8ad001a9ad56e94d2f3dc053822dae Mon Sep 17 00:00:00 2001 From: Kim Doyoung Date: Sun, 1 Apr 2018 21:32:14 +0900 Subject: [PATCH 34/34] Add `final` on Argument statement --- .../wake/remote/transport/netty/NettyMessagingTransport.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyMessagingTransport.java b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyMessagingTransport.java index a2a540f6d2..9dbd3c4882 100644 --- a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyMessagingTransport.java +++ b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/transport/netty/NettyMessagingTransport.java @@ -138,7 +138,7 @@ private NettyMessagingTransport( if (protocolType == ProtocolType.HTTP) { try{ this.uri = URI.create("http://" + host); - } catch (IllegalArgumentException e){ + } catch (final IllegalArgumentException e){ throw new RemoteRuntimeException("Invalid host address: " + host, e); } } else {